2024-11-24 22:13:38,637 main DEBUG Apache Log4j Core 2.17.2 initializing configuration org.apache.logging.log4j.core.config.properties.PropertiesConfiguration@47caedad 2024-11-24 22:13:38,647 main DEBUG Took 0.009061 seconds to load 1 plugins from package org.apache.hadoop.hbase.logging 2024-11-24 22:13:38,648 main DEBUG PluginManager 'Core' found 129 plugins 2024-11-24 22:13:38,648 main DEBUG PluginManager 'Level' found 0 plugins 2024-11-24 22:13:38,649 main DEBUG PluginManager 'Lookup' found 16 plugins 2024-11-24 22:13:38,650 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-24 22:13:38,656 main DEBUG PluginManager 'TypeConverter' found 26 plugins 2024-11-24 22:13:38,668 main DEBUG LoggerConfig$Builder(additivity="null", level="ERROR", levelAndRefs="null", name="org.apache.hadoop.metrics2.util.MBeans", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-24 22:13:38,669 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-24 22:13:38,669 main DEBUG LoggerConfig$Builder(additivity="null", level="DEBUG", levelAndRefs="null", name="org.apache.hadoop.hbase.logging.TestJul2Slf4j", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-24 22:13:38,670 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-24 22:13:38,670 main DEBUG LoggerConfig$Builder(additivity="null", level="ERROR", levelAndRefs="null", name="org.apache.zookeeper", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-24 22:13:38,670 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-24 22:13:38,671 main DEBUG LoggerConfig$Builder(additivity="null", level="WARN", levelAndRefs="null", name="org.apache.hadoop.metrics2.impl.MetricsSinkAdapter", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-24 22:13:38,671 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-24 22:13:38,672 main DEBUG LoggerConfig$Builder(additivity="null", level="ERROR", levelAndRefs="null", name="org.apache.hadoop.metrics2.impl.MetricsSystemImpl", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-24 22:13:38,672 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-24 22:13:38,673 main DEBUG LoggerConfig$Builder(additivity="false", level="WARN", levelAndRefs="null", name="org.apache.directory", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-24 22:13:38,673 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-24 22:13:38,673 main DEBUG LoggerConfig$Builder(additivity="null", level="DEBUG", levelAndRefs="null", name="org.apache.hadoop.hbase.ipc.FailedServers", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-24 22:13:38,674 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-24 22:13:38,674 main DEBUG LoggerConfig$Builder(additivity="null", level="WARN", levelAndRefs="null", name="org.apache.hadoop.metrics2.impl.MetricsConfig", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-24 22:13:38,674 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-24 22:13:38,675 main DEBUG LoggerConfig$Builder(additivity="null", level="INFO", levelAndRefs="null", name="org.apache.hadoop.hbase.ScheduledChore", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-24 22:13:38,675 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-24 22:13:38,675 main DEBUG LoggerConfig$Builder(additivity="null", level="DEBUG", levelAndRefs="null", name="org.apache.hadoop.hbase.regionserver.RSRpcServices", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-24 22:13:38,676 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-24 22:13:38,676 main DEBUG LoggerConfig$Builder(additivity="null", level="WARN", levelAndRefs="null", name="org.apache.hadoop", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-24 22:13:38,676 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-24 22:13:38,677 main DEBUG LoggerConfig$Builder(additivity="null", level="DEBUG", levelAndRefs="null", name="org.apache.hadoop.hbase", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-24 22:13:38,677 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-24 22:13:38,677 main DEBUG LoggerConfig$Builder(additivity="null", level="DEBUG", levelAndRefs="null", name="org.apache.hbase.thirdparty.io.netty.channel", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-24 22:13:38,678 main DEBUG Building Plugin[name=root, class=org.apache.logging.log4j.core.config.LoggerConfig$RootLogger]. 2024-11-24 22:13:38,679 main DEBUG LoggerConfig$RootLogger$Builder(additivity="null", level="null", levelAndRefs="INFO,Console", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-24 22:13:38,680 main DEBUG Building Plugin[name=loggers, class=org.apache.logging.log4j.core.config.LoggersPlugin]. 2024-11-24 22:13:38,681 main DEBUG createLoggers(={org.apache.hadoop.metrics2.util.MBeans, org.apache.hadoop.hbase.logging.TestJul2Slf4j, org.apache.zookeeper, org.apache.hadoop.metrics2.impl.MetricsSinkAdapter, org.apache.hadoop.metrics2.impl.MetricsSystemImpl, org.apache.directory, org.apache.hadoop.hbase.ipc.FailedServers, org.apache.hadoop.metrics2.impl.MetricsConfig, org.apache.hadoop.hbase.ScheduledChore, org.apache.hadoop.hbase.regionserver.RSRpcServices, org.apache.hadoop, org.apache.hadoop.hbase, org.apache.hbase.thirdparty.io.netty.channel, root}) 2024-11-24 22:13:38,682 main DEBUG Building Plugin[name=layout, class=org.apache.logging.log4j.core.layout.PatternLayout]. 2024-11-24 22:13:38,683 main DEBUG PatternLayout$Builder(pattern="%d{ISO8601} %-5p [%t%notEmpty{ %X}] %C{2}(%L): %m%n", PatternSelector=null, Configuration(PropertiesConfig), Replace=null, charset="null", alwaysWriteExceptions="null", disableAnsi="null", noConsoleNoAnsi="null", header="null", footer="null") 2024-11-24 22:13:38,683 main DEBUG PluginManager 'Converter' found 47 plugins 2024-11-24 22:13:38,691 main DEBUG Building Plugin[name=appender, class=org.apache.hadoop.hbase.logging.HBaseTestAppender]. 2024-11-24 22:13:38,693 main DEBUG HBaseTestAppender$Builder(target="SYSTEM_ERR", maxSize="1G", bufferedIo="null", bufferSize="null", immediateFlush="null", ignoreExceptions="null", PatternLayout(%d{ISO8601} %-5p [%t%notEmpty{ %X}] %C{2}(%L): %m%n), name="Console", Configuration(PropertiesConfig), Filter=null, ={}) 2024-11-24 22:13:38,694 main DEBUG Starting HBaseTestOutputStreamManager SYSTEM_ERR 2024-11-24 22:13:38,694 main DEBUG Building Plugin[name=appenders, class=org.apache.logging.log4j.core.config.AppendersPlugin]. 2024-11-24 22:13:38,695 main DEBUG createAppenders(={Console}) 2024-11-24 22:13:38,695 main DEBUG Configuration org.apache.logging.log4j.core.config.properties.PropertiesConfiguration@47caedad initialized 2024-11-24 22:13:38,696 main DEBUG Starting configuration org.apache.logging.log4j.core.config.properties.PropertiesConfiguration@47caedad 2024-11-24 22:13:38,696 main DEBUG Started configuration org.apache.logging.log4j.core.config.properties.PropertiesConfiguration@47caedad OK. 2024-11-24 22:13:38,696 main DEBUG Shutting down OutputStreamManager SYSTEM_OUT.false.false-1 2024-11-24 22:13:38,697 main DEBUG OutputStream closed 2024-11-24 22:13:38,697 main DEBUG Shut down OutputStreamManager SYSTEM_OUT.false.false-1, all resources released: true 2024-11-24 22:13:38,697 main DEBUG Appender DefaultConsole-1 stopped with status true 2024-11-24 22:13:38,697 main DEBUG Stopped org.apache.logging.log4j.core.config.DefaultConfiguration@55634720 OK 2024-11-24 22:13:38,763 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6 2024-11-24 22:13:38,765 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=StatusLogger 2024-11-24 22:13:38,766 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=ContextSelector 2024-11-24 22:13:38,767 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name= 2024-11-24 22:13:38,767 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.directory 2024-11-24 22:13:38,768 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.metrics2.impl.MetricsSinkAdapter 2024-11-24 22:13:38,768 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.zookeeper 2024-11-24 22:13:38,768 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.hbase.logging.TestJul2Slf4j 2024-11-24 22:13:38,768 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.metrics2.impl.MetricsSystemImpl 2024-11-24 22:13:38,769 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.metrics2.util.MBeans 2024-11-24 22:13:38,769 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.hbase 2024-11-24 22:13:38,769 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop 2024-11-24 22:13:38,770 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.hbase.ipc.FailedServers 2024-11-24 22:13:38,770 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.hbase.regionserver.RSRpcServices 2024-11-24 22:13:38,770 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.metrics2.impl.MetricsConfig 2024-11-24 22:13:38,770 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hbase.thirdparty.io.netty.channel 2024-11-24 22:13:38,771 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.hbase.ScheduledChore 2024-11-24 22:13:38,771 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Appenders,name=Console 2024-11-24 22:13:38,774 main DEBUG org.apache.logging.log4j.core.util.SystemClock supports precise timestamps. 2024-11-24 22:13:38,774 main DEBUG Reconfiguration complete for context[name=1dbd16a6] at URI jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-logging/target/hbase-logging-3.0.0-beta-2-SNAPSHOT-tests.jar!/log4j2.properties (org.apache.logging.log4j.core.LoggerContext@19835e64) with optional ClassLoader: null 2024-11-24 22:13:38,774 main DEBUG Shutdown hook enabled. Registering a new one. 2024-11-24 22:13:38,775 main DEBUG LoggerContext[name=1dbd16a6, org.apache.logging.log4j.core.LoggerContext@19835e64] started OK. 2024-11-24T22:13:39,014 DEBUG [main {}] hbase.HBaseTestingUtil(323): Setting hbase.rootdir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/c25542eb-68a7-66d5-8606-05e68b15f363 2024-11-24 22:13:39,018 main DEBUG AsyncLogger.ThreadNameStrategy=UNCACHED (user specified null, default is UNCACHED) 2024-11-24 22:13:39,018 main DEBUG org.apache.logging.log4j.core.util.SystemClock supports precise timestamps. 2024-11-24T22:13:39,027 INFO [main {}] hbase.HBaseClassTestRule(94): Test class org.apache.hadoop.hbase.TestAcidGuaranteesWithEagerPolicy timeout: 13 mins 2024-11-24T22:13:39,045 INFO [Time-limited test {}] hbase.HBaseTestingUtil(805): Starting up minicluster with option: StartMiniClusterOption{numMasters=1, masterClass=null, numRegionServers=1, rsPorts=, rsClass=null, numDataNodes=1, dataNodeHosts=null, numZkServers=1, createRootDir=false, createWALDir=false} 2024-11-24T22:13:39,060 INFO [Time-limited test {}] hbase.HBaseZKTestingUtil(84): Created new mini-cluster data directory: /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/c25542eb-68a7-66d5-8606-05e68b15f363/cluster_c8a06051-0388-dc48-080e-6657ef7cdb74, deleteOnExit=true 2024-11-24T22:13:39,060 INFO [Time-limited test {}] hbase.HBaseTestingUtil(818): STARTING DFS 2024-11-24T22:13:39,061 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting test.cache.data to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/c25542eb-68a7-66d5-8606-05e68b15f363/test.cache.data in system properties and HBase conf 2024-11-24T22:13:39,062 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting hadoop.tmp.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/c25542eb-68a7-66d5-8606-05e68b15f363/hadoop.tmp.dir in system properties and HBase conf 2024-11-24T22:13:39,062 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting hadoop.log.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/c25542eb-68a7-66d5-8606-05e68b15f363/hadoop.log.dir in system properties and HBase conf 2024-11-24T22:13:39,063 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting mapreduce.cluster.local.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/c25542eb-68a7-66d5-8606-05e68b15f363/mapreduce.cluster.local.dir in system properties and HBase conf 2024-11-24T22:13:39,063 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting mapreduce.cluster.temp.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/c25542eb-68a7-66d5-8606-05e68b15f363/mapreduce.cluster.temp.dir in system properties and HBase conf 2024-11-24T22:13:39,064 INFO [Time-limited test {}] hbase.HBaseTestingUtil(738): read short circuit is OFF 2024-11-24T22:13:39,149 WARN [Time-limited test {}] util.NativeCodeLoader(60): Unable to load native-hadoop library for your platform... using builtin-java classes where applicable 2024-11-24T22:13:39,239 DEBUG [Time-limited test {}] fs.HFileSystem(310): The file system is not a DistributedFileSystem. Skipping on block location reordering 2024-11-24T22:13:39,243 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.node-labels.fs-store.root-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/c25542eb-68a7-66d5-8606-05e68b15f363/yarn.node-labels.fs-store.root-dir in system properties and HBase conf 2024-11-24T22:13:39,243 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.node-attribute.fs-store.root-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/c25542eb-68a7-66d5-8606-05e68b15f363/yarn.node-attribute.fs-store.root-dir in system properties and HBase conf 2024-11-24T22:13:39,244 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.nodemanager.log-dirs to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/c25542eb-68a7-66d5-8606-05e68b15f363/yarn.nodemanager.log-dirs in system properties and HBase conf 2024-11-24T22:13:39,244 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.nodemanager.remote-app-log-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/c25542eb-68a7-66d5-8606-05e68b15f363/yarn.nodemanager.remote-app-log-dir in system properties and HBase conf 2024-11-24T22:13:39,245 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.timeline-service.entity-group-fs-store.active-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/c25542eb-68a7-66d5-8606-05e68b15f363/yarn.timeline-service.entity-group-fs-store.active-dir in system properties and HBase conf 2024-11-24T22:13:39,245 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.timeline-service.entity-group-fs-store.done-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/c25542eb-68a7-66d5-8606-05e68b15f363/yarn.timeline-service.entity-group-fs-store.done-dir in system properties and HBase conf 2024-11-24T22:13:39,245 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.nodemanager.remote-app-log-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/c25542eb-68a7-66d5-8606-05e68b15f363/yarn.nodemanager.remote-app-log-dir in system properties and HBase conf 2024-11-24T22:13:39,246 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting dfs.journalnode.edits.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/c25542eb-68a7-66d5-8606-05e68b15f363/dfs.journalnode.edits.dir in system properties and HBase conf 2024-11-24T22:13:39,246 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting dfs.datanode.shared.file.descriptor.paths to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/c25542eb-68a7-66d5-8606-05e68b15f363/dfs.datanode.shared.file.descriptor.paths in system properties and HBase conf 2024-11-24T22:13:39,246 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting nfs.dump.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/c25542eb-68a7-66d5-8606-05e68b15f363/nfs.dump.dir in system properties and HBase conf 2024-11-24T22:13:39,247 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting java.io.tmpdir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/c25542eb-68a7-66d5-8606-05e68b15f363/java.io.tmpdir in system properties and HBase conf 2024-11-24T22:13:39,247 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting dfs.journalnode.edits.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/c25542eb-68a7-66d5-8606-05e68b15f363/dfs.journalnode.edits.dir in system properties and HBase conf 2024-11-24T22:13:39,248 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting dfs.provided.aliasmap.inmemory.leveldb.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/c25542eb-68a7-66d5-8606-05e68b15f363/dfs.provided.aliasmap.inmemory.leveldb.dir in system properties and HBase conf 2024-11-24T22:13:39,248 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting fs.s3a.committer.staging.tmp.path to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/c25542eb-68a7-66d5-8606-05e68b15f363/fs.s3a.committer.staging.tmp.path in system properties and HBase conf 2024-11-24T22:13:40,170 WARN [Time-limited test {}] impl.MetricsConfig(138): Cannot locate configuration: tried hadoop-metrics2-namenode.properties,hadoop-metrics2.properties 2024-11-24T22:13:40,254 INFO [Time-limited test {}] log.Log(170): Logging initialized @2375ms to org.eclipse.jetty.util.log.Slf4jLog 2024-11-24T22:13:40,330 WARN [Time-limited test {}] server.AuthenticationFilter(240): Unable to initialize FileSignerSecretProvider, falling back to use random secrets. Reason: Could not read signature secret file: /home/jenkins/hadoop-http-auth-signature-secret 2024-11-24T22:13:40,396 INFO [Time-limited test {}] server.Server(375): jetty-9.4.53.v20231009; built: 2023-10-09T12:29:09.265Z; git: 27bde00a0b95a1d5bbee0eae7984f891d2d0f8c9; jvm 17.0.11+9 2024-11-24T22:13:40,415 INFO [Time-limited test {}] session.DefaultSessionIdManager(334): DefaultSessionIdManager workerName=node0 2024-11-24T22:13:40,415 INFO [Time-limited test {}] session.DefaultSessionIdManager(339): No SessionScavenger set, using defaults 2024-11-24T22:13:40,417 INFO [Time-limited test {}] session.HouseKeeper(132): node0 Scavenging every 660000ms 2024-11-24T22:13:40,429 WARN [Time-limited test {}] server.AuthenticationFilter(240): Unable to initialize FileSignerSecretProvider, falling back to use random secrets. Reason: Could not read signature secret file: /home/jenkins/hadoop-http-auth-signature-secret 2024-11-24T22:13:40,431 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@6b23711f{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/c25542eb-68a7-66d5-8606-05e68b15f363/hadoop.log.dir/,AVAILABLE} 2024-11-24T22:13:40,432 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@93c62df{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,AVAILABLE} 2024-11-24T22:13:40,668 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.w.WebAppContext@4ded99d3{hdfs,/,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/c25542eb-68a7-66d5-8606-05e68b15f363/java.io.tmpdir/jetty-localhost-33955-hadoop-hdfs-3_4_1-tests_jar-_-any-16701226721670210477/webapp/,AVAILABLE}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/hdfs} 2024-11-24T22:13:40,679 INFO [Time-limited test {}] server.AbstractConnector(333): Started ServerConnector@557285fa{HTTP/1.1, (http/1.1)}{localhost:33955} 2024-11-24T22:13:40,679 INFO [Time-limited test {}] server.Server(415): Started @2800ms 2024-11-24T22:13:41,197 WARN [Time-limited test {}] server.AuthenticationFilter(240): Unable to initialize FileSignerSecretProvider, falling back to use random secrets. Reason: Could not read signature secret file: /home/jenkins/hadoop-http-auth-signature-secret 2024-11-24T22:13:41,203 INFO [Time-limited test {}] server.Server(375): jetty-9.4.53.v20231009; built: 2023-10-09T12:29:09.265Z; git: 27bde00a0b95a1d5bbee0eae7984f891d2d0f8c9; jvm 17.0.11+9 2024-11-24T22:13:41,204 INFO [Time-limited test {}] session.DefaultSessionIdManager(334): DefaultSessionIdManager workerName=node0 2024-11-24T22:13:41,205 INFO [Time-limited test {}] session.DefaultSessionIdManager(339): No SessionScavenger set, using defaults 2024-11-24T22:13:41,205 INFO [Time-limited test {}] session.HouseKeeper(132): node0 Scavenging every 600000ms 2024-11-24T22:13:41,206 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@4fd7fa7c{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/c25542eb-68a7-66d5-8606-05e68b15f363/hadoop.log.dir/,AVAILABLE} 2024-11-24T22:13:41,207 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@435f9f96{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,AVAILABLE} 2024-11-24T22:13:41,331 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.w.WebAppContext@4269bd31{datanode,/,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/c25542eb-68a7-66d5-8606-05e68b15f363/java.io.tmpdir/jetty-localhost-39987-hadoop-hdfs-3_4_1-tests_jar-_-any-3077355853976122194/webapp/,AVAILABLE}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/datanode} 2024-11-24T22:13:41,332 INFO [Time-limited test {}] server.AbstractConnector(333): Started ServerConnector@353d64f2{HTTP/1.1, (http/1.1)}{localhost:39987} 2024-11-24T22:13:41,332 INFO [Time-limited test {}] server.Server(415): Started @3453ms 2024-11-24T22:13:41,389 WARN [Time-limited test {}] web.RestCsrfPreventionFilterHandler(75): Got null for restCsrfPreventionFilter - will not do any filtering. 2024-11-24T22:13:41,931 WARN [Thread-72 {}] impl.BlockPoolSlice(347): dfsUsed file missing in /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/c25542eb-68a7-66d5-8606-05e68b15f363/cluster_c8a06051-0388-dc48-080e-6657ef7cdb74/data/data1/current/BP-2007615641-172.17.0.2-1732486419886/current, will proceed with Du for space computation calculation, 2024-11-24T22:13:41,931 WARN [Thread-73 {}] impl.BlockPoolSlice(347): dfsUsed file missing in /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/c25542eb-68a7-66d5-8606-05e68b15f363/cluster_c8a06051-0388-dc48-080e-6657ef7cdb74/data/data2/current/BP-2007615641-172.17.0.2-1732486419886/current, will proceed with Du for space computation calculation, 2024-11-24T22:13:41,963 WARN [Thread-58 {}] datanode.DirectoryScanner(302): dfs.datanode.directoryscan.throttle.limit.ms.per.sec set to value above 1000 ms/sec. Assuming default value of -1 2024-11-24T22:13:42,019 DEBUG [Time-limited test {}] hbase.HBaseTestingUtil(631): Setting hbase.rootdir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/c25542eb-68a7-66d5-8606-05e68b15f363 2024-11-24T22:13:42,021 INFO [Block report processor {}] blockmanagement.BlockManager(2940): BLOCK* processReport 0xcec937cb90161702 with lease ID 0xa4ad5e23ed6cbcbb: Processing first storage report for DS-37016b53-75fb-429c-b62c-a33fe5baf0d3 from datanode DatanodeRegistration(127.0.0.1:45929, datanodeUuid=1c5ad8e8-4392-4afa-9943-4f874d844787, infoPort=41971, infoSecurePort=0, ipcPort=46399, storageInfo=lv=-57;cid=testClusterID;nsid=26724100;c=1732486419886) 2024-11-24T22:13:42,022 INFO [Block report processor {}] blockmanagement.BlockManager(2972): BLOCK* processReport 0xcec937cb90161702 with lease ID 0xa4ad5e23ed6cbcbb: from storage DS-37016b53-75fb-429c-b62c-a33fe5baf0d3 node DatanodeRegistration(127.0.0.1:45929, datanodeUuid=1c5ad8e8-4392-4afa-9943-4f874d844787, infoPort=41971, infoSecurePort=0, ipcPort=46399, storageInfo=lv=-57;cid=testClusterID;nsid=26724100;c=1732486419886), blocks: 0, hasStaleStorage: true, processing time: 2 msecs, invalidatedBlocks: 0 2024-11-24T22:13:42,022 INFO [Block report processor {}] blockmanagement.BlockManager(2940): BLOCK* processReport 0xcec937cb90161702 with lease ID 0xa4ad5e23ed6cbcbb: Processing first storage report for DS-4c3a0dae-a8ad-48c7-b78b-4de2d7debf46 from datanode DatanodeRegistration(127.0.0.1:45929, datanodeUuid=1c5ad8e8-4392-4afa-9943-4f874d844787, infoPort=41971, infoSecurePort=0, ipcPort=46399, storageInfo=lv=-57;cid=testClusterID;nsid=26724100;c=1732486419886) 2024-11-24T22:13:42,023 INFO [Block report processor {}] blockmanagement.BlockManager(2972): BLOCK* processReport 0xcec937cb90161702 with lease ID 0xa4ad5e23ed6cbcbb: from storage DS-4c3a0dae-a8ad-48c7-b78b-4de2d7debf46 node DatanodeRegistration(127.0.0.1:45929, datanodeUuid=1c5ad8e8-4392-4afa-9943-4f874d844787, infoPort=41971, infoSecurePort=0, ipcPort=46399, storageInfo=lv=-57;cid=testClusterID;nsid=26724100;c=1732486419886), blocks: 0, hasStaleStorage: false, processing time: 0 msecs, invalidatedBlocks: 0 2024-11-24T22:13:42,104 INFO [Time-limited test {}] zookeeper.MiniZooKeeperCluster(261): Started connectionTimeout=30000, dir=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/c25542eb-68a7-66d5-8606-05e68b15f363/cluster_c8a06051-0388-dc48-080e-6657ef7cdb74/zookeeper_0, clientPort=59994, secureClientPort=-1, dataDir=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/c25542eb-68a7-66d5-8606-05e68b15f363/cluster_c8a06051-0388-dc48-080e-6657ef7cdb74/zookeeper_0/version-2, dataDirSize=457 dataLogDir=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/c25542eb-68a7-66d5-8606-05e68b15f363/cluster_c8a06051-0388-dc48-080e-6657ef7cdb74/zookeeper_0/version-2, dataLogSize=457 tickTime=2000, maxClientCnxns=300, minSessionTimeout=4000, maxSessionTimeout=40000, clientPortListenBacklog=-1, serverId=0 2024-11-24T22:13:42,113 INFO [Time-limited test {}] zookeeper.MiniZooKeeperCluster(286): Started MiniZooKeeperCluster and ran 'stat' on client port=59994 2024-11-24T22:13:42,123 INFO [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-11-24T22:13:42,126 INFO [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-11-24T22:13:42,364 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45929 is added to blk_1073741825_1001 (size=7) 2024-11-24T22:13:42,774 INFO [Time-limited test {}] util.FSUtils(489): Created version file at hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc with version=8 2024-11-24T22:13:42,775 INFO [Time-limited test {}] hbase.HBaseTestingUtil(1139): Setting hbase.fs.tmp.dir to hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/hbase-staging 2024-11-24T22:13:42,912 DEBUG [Time-limited test {}] channel.MultithreadEventLoopGroup(44): -Dio.netty.eventLoopThreads: 16 2024-11-24T22:13:43,202 INFO [Time-limited test {}] client.ConnectionUtils(128): master/bd6592c37bb2:0 server-side Connection retries=45 2024-11-24T22:13:43,212 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated default.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-11-24T22:13:43,213 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated priority.RWQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=2, maxQueueLength=30, handlerCount=3 2024-11-24T22:13:43,217 INFO [Time-limited test {}] ipc.RWQueueRpcExecutor(113): priority.RWQ.Fifo writeQueues=1 writeHandlers=1 readQueues=1 readHandlers=2 scanQueues=0 scanHandlers=0 2024-11-24T22:13:43,217 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated replication.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-11-24T22:13:43,217 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated metaPriority.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=1 2024-11-24T22:13:43,357 INFO [Time-limited test {}] ipc.RpcServerFactory(64): Creating org.apache.hadoop.hbase.ipc.NettyRpcServer hosting hbase.pb.MasterService, hbase.pb.RegionServerStatusService, hbase.pb.LockService, hbase.pb.HbckService, hbase.pb.ClientMetaService, hbase.pb.AdminService 2024-11-24T22:13:43,416 INFO [Time-limited test {}] metrics.MetricRegistriesLoader(60): Loaded MetricRegistries class org.apache.hadoop.hbase.metrics.impl.MetricRegistriesImpl 2024-11-24T22:13:43,424 DEBUG [Time-limited test {}] util.ClassSize(228): Using Unsafe to estimate memory layout 2024-11-24T22:13:43,428 INFO [Time-limited test {}] ipc.NettyRpcServer(309): Using org.apache.hbase.thirdparty.io.netty.buffer.PooledByteBufAllocator for buffer allocation 2024-11-24T22:13:43,454 DEBUG [Time-limited test {}] channel.DefaultChannelId(84): -Dio.netty.processId: 3079 (auto-detected) 2024-11-24T22:13:43,455 DEBUG [Time-limited test {}] channel.DefaultChannelId(106): -Dio.netty.machineId: 02:42:ac:ff:fe:11:00:02 (auto-detected) 2024-11-24T22:13:43,475 INFO [Time-limited test {}] ipc.NettyRpcServer(191): Bind to /172.17.0.2:37701 2024-11-24T22:13:43,506 INFO [Time-limited test {}] zookeeper.RecoverableZooKeeper(137): Process identifier=master:37701 connecting to ZooKeeper ensemble=127.0.0.1:59994 2024-11-24T22:13:43,547 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:377010x0, quorum=127.0.0.1:59994, baseZNode=/hbase Received ZooKeeper Event, type=None, state=SyncConnected, path=null 2024-11-24T22:13:43,551 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKWatcher(635): master:37701-0x1014c0ac8d60000 connected 2024-11-24T22:13:43,589 INFO [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-11-24T22:13:43,592 INFO [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-11-24T22:13:43,603 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): master:37701-0x1014c0ac8d60000, quorum=127.0.0.1:59994, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running 2024-11-24T22:13:43,607 INFO [Time-limited test {}] master.HMaster(525): hbase.rootdir=hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc, hbase.cluster.distributed=false 2024-11-24T22:13:43,629 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): master:37701-0x1014c0ac8d60000, quorum=127.0.0.1:59994, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/acl 2024-11-24T22:13:43,633 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=default.FPBQ.Fifo, numCallQueues=1, port=37701 2024-11-24T22:13:43,634 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=priority.RWQ.Fifo.write, numCallQueues=1, port=37701 2024-11-24T22:13:43,636 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=priority.RWQ.Fifo.read, numCallQueues=1, port=37701 2024-11-24T22:13:43,637 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=replication.FPBQ.Fifo, numCallQueues=1, port=37701 2024-11-24T22:13:43,637 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=metaPriority.FPBQ.Fifo, numCallQueues=1, port=37701 2024-11-24T22:13:43,752 INFO [Time-limited test {}] client.ConnectionUtils(128): regionserver/bd6592c37bb2:0 server-side Connection retries=45 2024-11-24T22:13:43,754 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated default.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-11-24T22:13:43,754 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated priority.RWQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=2, maxQueueLength=30, handlerCount=3 2024-11-24T22:13:43,755 INFO [Time-limited test {}] ipc.RWQueueRpcExecutor(113): priority.RWQ.Fifo writeQueues=1 writeHandlers=1 readQueues=1 readHandlers=2 scanQueues=0 scanHandlers=0 2024-11-24T22:13:43,755 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated replication.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-11-24T22:13:43,755 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated metaPriority.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=1 2024-11-24T22:13:43,758 INFO [Time-limited test {}] ipc.RpcServerFactory(64): Creating org.apache.hadoop.hbase.ipc.NettyRpcServer hosting hbase.pb.ClientService, hbase.pb.AdminService, hbase.pb.ClientMetaService, hbase.pb.BootstrapNodeService 2024-11-24T22:13:43,761 INFO [Time-limited test {}] ipc.NettyRpcServer(309): Using org.apache.hbase.thirdparty.io.netty.buffer.PooledByteBufAllocator for buffer allocation 2024-11-24T22:13:43,762 INFO [Time-limited test {}] ipc.NettyRpcServer(191): Bind to /172.17.0.2:40087 2024-11-24T22:13:43,764 INFO [Time-limited test {}] zookeeper.RecoverableZooKeeper(137): Process identifier=regionserver:40087 connecting to ZooKeeper ensemble=127.0.0.1:59994 2024-11-24T22:13:43,765 INFO [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-11-24T22:13:43,768 INFO [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-11-24T22:13:43,778 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:400870x0, quorum=127.0.0.1:59994, baseZNode=/hbase Received ZooKeeper Event, type=None, state=SyncConnected, path=null 2024-11-24T22:13:43,779 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKWatcher(635): regionserver:40087-0x1014c0ac8d60001 connected 2024-11-24T22:13:43,779 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:40087-0x1014c0ac8d60001, quorum=127.0.0.1:59994, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running 2024-11-24T22:13:43,785 INFO [Time-limited test {}] hfile.BlockCacheFactory(123): Allocating BlockCache size=880 MB, blockSize=64 KB 2024-11-24T22:13:43,794 DEBUG [Time-limited test {}] mob.MobFileCache(124): MobFileCache enabled with cacheSize=1000, evictPeriods=3600sec, evictRemainRatio=0.5 2024-11-24T22:13:43,797 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:40087-0x1014c0ac8d60001, quorum=127.0.0.1:59994, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/master 2024-11-24T22:13:43,802 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:40087-0x1014c0ac8d60001, quorum=127.0.0.1:59994, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/acl 2024-11-24T22:13:43,802 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=default.FPBQ.Fifo, numCallQueues=1, port=40087 2024-11-24T22:13:43,802 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=priority.RWQ.Fifo.write, numCallQueues=1, port=40087 2024-11-24T22:13:43,803 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=priority.RWQ.Fifo.read, numCallQueues=1, port=40087 2024-11-24T22:13:43,804 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=replication.FPBQ.Fifo, numCallQueues=1, port=40087 2024-11-24T22:13:43,807 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=metaPriority.FPBQ.Fifo, numCallQueues=1, port=40087 2024-11-24T22:13:43,822 DEBUG [M:0;bd6592c37bb2:37701 {}] regionserver.ShutdownHook(81): Installed shutdown hook thread: Shutdownhook:M:0;bd6592c37bb2:37701 2024-11-24T22:13:43,823 INFO [master/bd6592c37bb2:0:becomeActiveMaster {}] master.HMaster(2510): Adding backup master ZNode /hbase/backup-masters/bd6592c37bb2,37701,1732486422992 2024-11-24T22:13:43,830 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:40087-0x1014c0ac8d60001, quorum=127.0.0.1:59994, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters 2024-11-24T22:13:43,830 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:37701-0x1014c0ac8d60000, quorum=127.0.0.1:59994, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters 2024-11-24T22:13:43,832 DEBUG [master/bd6592c37bb2:0:becomeActiveMaster {}] zookeeper.ZKUtil(111): master:37701-0x1014c0ac8d60000, quorum=127.0.0.1:59994, baseZNode=/hbase Set watcher on existing znode=/hbase/backup-masters/bd6592c37bb2,37701,1732486422992 2024-11-24T22:13:43,852 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:40087-0x1014c0ac8d60001, quorum=127.0.0.1:59994, baseZNode=/hbase Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/hbase/master 2024-11-24T22:13:43,852 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:37701-0x1014c0ac8d60000, quorum=127.0.0.1:59994, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-24T22:13:43,852 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:40087-0x1014c0ac8d60001, quorum=127.0.0.1:59994, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-24T22:13:43,853 DEBUG [master/bd6592c37bb2:0:becomeActiveMaster {}] zookeeper.ZKUtil(111): master:37701-0x1014c0ac8d60000, quorum=127.0.0.1:59994, baseZNode=/hbase Set watcher on existing znode=/hbase/master 2024-11-24T22:13:43,854 INFO [master/bd6592c37bb2:0:becomeActiveMaster {}] master.ActiveMasterManager(245): Deleting ZNode for /hbase/backup-masters/bd6592c37bb2,37701,1732486422992 from backup master directory 2024-11-24T22:13:43,857 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:37701-0x1014c0ac8d60000, quorum=127.0.0.1:59994, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/backup-masters/bd6592c37bb2,37701,1732486422992 2024-11-24T22:13:43,857 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:40087-0x1014c0ac8d60001, quorum=127.0.0.1:59994, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters 2024-11-24T22:13:43,857 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:37701-0x1014c0ac8d60000, quorum=127.0.0.1:59994, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters 2024-11-24T22:13:43,858 WARN [master/bd6592c37bb2:0:becomeActiveMaster {}] hbase.ZNodeClearer(65): Environment variable HBASE_ZNODE_FILE not set; znodes will not be cleared on crash by start scripts (Longer MTTR!) 2024-11-24T22:13:43,858 INFO [master/bd6592c37bb2:0:becomeActiveMaster {}] master.ActiveMasterManager(255): Registered as active master=bd6592c37bb2,37701,1732486422992 2024-11-24T22:13:43,860 INFO [master/bd6592c37bb2:0:becomeActiveMaster {}] regionserver.ChunkCreator(488): Allocating data MemStoreChunkPool with chunk size 2 MB, max count 396, initial count 0 2024-11-24T22:13:43,862 INFO [master/bd6592c37bb2:0:becomeActiveMaster {}] regionserver.ChunkCreator(488): Allocating index MemStoreChunkPool with chunk size 204.80 KB, max count 440, initial count 0 2024-11-24T22:13:43,916 DEBUG [master/bd6592c37bb2:0:becomeActiveMaster {}] util.FSUtils(620): Create cluster ID file [hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/hbase.id] with ID: 3c461ac1-06fb-4739-8702-6368e685b03c 2024-11-24T22:13:43,916 DEBUG [master/bd6592c37bb2:0:becomeActiveMaster {}] util.FSUtils(625): Write the cluster ID file to a temporary location: hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/.tmp/hbase.id 2024-11-24T22:13:43,926 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45929 is added to blk_1073741826_1002 (size=42) 2024-11-24T22:13:43,927 DEBUG [master/bd6592c37bb2:0:becomeActiveMaster {}] util.FSUtils(634): Move the temporary cluster ID file to its target location [hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/.tmp/hbase.id]:[hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/hbase.id] 2024-11-24T22:13:43,966 INFO [master/bd6592c37bb2:0:becomeActiveMaster {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-11-24T22:13:43,971 INFO [master/bd6592c37bb2:0:becomeActiveMaster {}] util.FSTableDescriptors(270): Fetching table descriptors from the filesystem. 2024-11-24T22:13:43,990 INFO [master/bd6592c37bb2:0:becomeActiveMaster {}] util.FSTableDescriptors(299): Fetched table descriptors(size=0) cost 17ms. 2024-11-24T22:13:43,996 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:40087-0x1014c0ac8d60001, quorum=127.0.0.1:59994, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-24T22:13:43,996 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:37701-0x1014c0ac8d60000, quorum=127.0.0.1:59994, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-24T22:13:44,013 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45929 is added to blk_1073741827_1003 (size=196) 2024-11-24T22:13:44,038 INFO [master/bd6592c37bb2:0:becomeActiveMaster {}] region.MasterRegion(370): Create or load local region for table 'master:store', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'proc', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'rs', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'state', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-11-24T22:13:44,040 INFO [master/bd6592c37bb2:0:becomeActiveMaster {}] region.MasterRegionFlusherAndCompactor(132): Injected flushSize=134217728, flushPerChanges=1000000, flushIntervalMs=900000 2024-11-24T22:13:44,059 DEBUG [master/bd6592c37bb2:0:becomeActiveMaster {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(244): No decryptEncryptedDataEncryptionKey method in DFSClient, should be hadoop version with HDFS-12396 java.lang.NoSuchMethodException: org.apache.hadoop.hdfs.DFSClient.decryptEncryptedDataEncryptionKey(org.apache.hadoop.fs.FileEncryptionInfo) at java.lang.Class.getDeclaredMethod(Class.java:2675) ~[?:?] at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper.createTransparentCryptoHelperWithoutHDFS12396(FanOutOneBlockAsyncDFSOutputSaslHelper.java:183) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper.createTransparentCryptoHelper(FanOutOneBlockAsyncDFSOutputSaslHelper.java:242) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper.(FanOutOneBlockAsyncDFSOutputSaslHelper.java:253) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.lang.Class.forName0(Native Method) ~[?:?] at java.lang.Class.forName(Class.java:375) ~[?:?] at org.apache.hadoop.hbase.wal.AsyncFSWALProvider.load(AsyncFSWALProvider.java:150) ~[classes/:?] at org.apache.hadoop.hbase.wal.WALFactory.getProviderClass(WALFactory.java:174) ~[classes/:?] at org.apache.hadoop.hbase.wal.WALFactory.(WALFactory.java:262) ~[classes/:?] at org.apache.hadoop.hbase.wal.WALFactory.(WALFactory.java:231) ~[classes/:?] at org.apache.hadoop.hbase.master.region.MasterRegion.create(MasterRegion.java:400) ~[classes/:?] at org.apache.hadoop.hbase.master.region.MasterRegionFactory.create(MasterRegionFactory.java:135) ~[classes/:?] at org.apache.hadoop.hbase.master.HMaster.finishActiveMasterInitialization(HMaster.java:1003) ~[classes/:?] at org.apache.hadoop.hbase.master.HMaster.startActiveMasterManager(HMaster.java:2535) ~[classes/:?] at org.apache.hadoop.hbase.master.HMaster.lambda$run$0(HMaster.java:613) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.lambda$tracedRunnable$2(TraceUtil.java:155) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T22:13:44,064 INFO [master/bd6592c37bb2:0:becomeActiveMaster {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-11-24T22:13:44,100 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45929 is added to blk_1073741828_1004 (size=1189) 2024-11-24T22:13:44,122 INFO [master/bd6592c37bb2:0:becomeActiveMaster {}] regionserver.HRegion(7590): Creating {ENCODED => 1595e783b53d99cd5eef43b6debb2682, NAME => 'master:store,,1.1595e783b53d99cd5eef43b6debb2682.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='master:store', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'proc', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'rs', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'state', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, under table dir hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/MasterData/data/master/store 2024-11-24T22:13:44,144 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45929 is added to blk_1073741829_1005 (size=34) 2024-11-24T22:13:44,549 INFO [master/bd6592c37bb2:0:becomeActiveMaster {}] throttle.StoreHotnessProtector(112): StoreHotnessProtector is disabled. Set hbase.region.store.parallel.put.limit > 0 to enable, which may help mitigate load under heavy write pressure. 2024-11-24T22:13:44,552 DEBUG [master/bd6592c37bb2:0:becomeActiveMaster {}] regionserver.HRegion(898): Instantiated master:store,,1.1595e783b53d99cd5eef43b6debb2682.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-24T22:13:44,553 DEBUG [master/bd6592c37bb2:0:becomeActiveMaster {}] regionserver.HRegion(1722): Closing 1595e783b53d99cd5eef43b6debb2682, disabling compactions & flushes 2024-11-24T22:13:44,554 INFO [master/bd6592c37bb2:0:becomeActiveMaster {}] regionserver.HRegion(1755): Closing region master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-24T22:13:44,554 DEBUG [master/bd6592c37bb2:0:becomeActiveMaster {}] regionserver.HRegion(1776): Time limited wait for close lock on master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-24T22:13:44,555 DEBUG [master/bd6592c37bb2:0:becomeActiveMaster {}] regionserver.HRegion(1843): Acquired close lock on master:store,,1.1595e783b53d99cd5eef43b6debb2682. after waiting 0 ms 2024-11-24T22:13:44,555 DEBUG [master/bd6592c37bb2:0:becomeActiveMaster {}] regionserver.HRegion(1853): Updates disabled for region master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-24T22:13:44,555 INFO [master/bd6592c37bb2:0:becomeActiveMaster {}] regionserver.HRegion(1973): Closed master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-24T22:13:44,556 DEBUG [master/bd6592c37bb2:0:becomeActiveMaster {}] regionserver.HRegion(1676): Region close journal for 1595e783b53d99cd5eef43b6debb2682: Waiting for close lock at 1732486424553Disabling compacts and flushes for region at 1732486424553Disabling writes for close at 1732486424555 (+2 ms)Writing region close event to WAL at 1732486424555Closed at 1732486424555 2024-11-24T22:13:44,558 WARN [master/bd6592c37bb2:0:becomeActiveMaster {}] region.MasterRegion(249): failed to clean up initializing flag: hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/MasterData/data/master/store/.initializing 2024-11-24T22:13:44,559 DEBUG [master/bd6592c37bb2:0:becomeActiveMaster {}] region.MasterRegion(219): WALDir=hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/MasterData/WALs/bd6592c37bb2,37701,1732486422992 2024-11-24T22:13:44,568 INFO [master/bd6592c37bb2:0:becomeActiveMaster {}] monitor.StreamSlowMonitor(122): New stream slow monitor defaultMonitorName 2024-11-24T22:13:44,582 INFO [master/bd6592c37bb2:0:becomeActiveMaster {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=bd6592c37bb2%2C37701%2C1732486422992, suffix=, logDir=hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/MasterData/WALs/bd6592c37bb2,37701,1732486422992, archiveDir=hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/MasterData/oldWALs, maxLogs=10 2024-11-24T22:13:44,605 DEBUG [master/bd6592c37bb2:0:becomeActiveMaster {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/MasterData/WALs/bd6592c37bb2,37701,1732486422992/bd6592c37bb2%2C37701%2C1732486422992.1732486424587, exclude list is [], retry=0 2024-11-24T22:13:44,620 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:45929,DS-37016b53-75fb-429c-b62c-a33fe5baf0d3,DISK] 2024-11-24T22:13:44,623 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] asyncfs.ProtobufDecoder(117): Hadoop 3.3 and above shades protobuf. 2024-11-24T22:13:44,659 INFO [master/bd6592c37bb2:0:becomeActiveMaster {}] wal.AbstractFSWAL(991): New WAL /user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/MasterData/WALs/bd6592c37bb2,37701,1732486422992/bd6592c37bb2%2C37701%2C1732486422992.1732486424587 2024-11-24T22:13:44,660 DEBUG [master/bd6592c37bb2:0:becomeActiveMaster {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:41971:41971)] 2024-11-24T22:13:44,660 DEBUG [master/bd6592c37bb2:0:becomeActiveMaster {}] regionserver.HRegion(7752): Opening region: {ENCODED => 1595e783b53d99cd5eef43b6debb2682, NAME => 'master:store,,1.1595e783b53d99cd5eef43b6debb2682.', STARTKEY => '', ENDKEY => ''} 2024-11-24T22:13:44,661 DEBUG [master/bd6592c37bb2:0:becomeActiveMaster {}] regionserver.HRegion(898): Instantiated master:store,,1.1595e783b53d99cd5eef43b6debb2682.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-24T22:13:44,664 DEBUG [master/bd6592c37bb2:0:becomeActiveMaster {}] regionserver.HRegion(7794): checking encryption for 1595e783b53d99cd5eef43b6debb2682 2024-11-24T22:13:44,665 DEBUG [master/bd6592c37bb2:0:becomeActiveMaster {}] regionserver.HRegion(7797): checking classloading for 1595e783b53d99cd5eef43b6debb2682 2024-11-24T22:13:44,701 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family info of region 1595e783b53d99cd5eef43b6debb2682 2024-11-24T22:13:44,727 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1595e783b53d99cd5eef43b6debb2682 columnFamilyName info 2024-11-24T22:13:44,730 DEBUG [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T22:13:44,733 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(327): Store=1595e783b53d99cd5eef43b6debb2682/info, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-24T22:13:44,733 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family proc of region 1595e783b53d99cd5eef43b6debb2682 2024-11-24T22:13:44,736 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1595e783b53d99cd5eef43b6debb2682 columnFamilyName proc 2024-11-24T22:13:44,736 DEBUG [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T22:13:44,737 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(327): Store=1595e783b53d99cd5eef43b6debb2682/proc, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-24T22:13:44,737 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family rs of region 1595e783b53d99cd5eef43b6debb2682 2024-11-24T22:13:44,740 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1595e783b53d99cd5eef43b6debb2682 columnFamilyName rs 2024-11-24T22:13:44,740 DEBUG [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T22:13:44,741 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(327): Store=1595e783b53d99cd5eef43b6debb2682/rs, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-24T22:13:44,741 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family state of region 1595e783b53d99cd5eef43b6debb2682 2024-11-24T22:13:44,743 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1595e783b53d99cd5eef43b6debb2682 columnFamilyName state 2024-11-24T22:13:44,744 DEBUG [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T22:13:44,744 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(327): Store=1595e783b53d99cd5eef43b6debb2682/state, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-24T22:13:44,745 DEBUG [master/bd6592c37bb2:0:becomeActiveMaster {}] regionserver.HRegion(1038): replaying wal for 1595e783b53d99cd5eef43b6debb2682 2024-11-24T22:13:44,748 DEBUG [master/bd6592c37bb2:0:becomeActiveMaster {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682 2024-11-24T22:13:44,749 DEBUG [master/bd6592c37bb2:0:becomeActiveMaster {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682 2024-11-24T22:13:44,754 DEBUG [master/bd6592c37bb2:0:becomeActiveMaster {}] regionserver.HRegion(1048): stopping wal replay for 1595e783b53d99cd5eef43b6debb2682 2024-11-24T22:13:44,755 DEBUG [master/bd6592c37bb2:0:becomeActiveMaster {}] regionserver.HRegion(1060): Cleaning up temporary data for 1595e783b53d99cd5eef43b6debb2682 2024-11-24T22:13:44,758 DEBUG [master/bd6592c37bb2:0:becomeActiveMaster {}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table master:store descriptor;using region.getMemStoreFlushHeapSize/# of families (32.0 M)) instead. 2024-11-24T22:13:44,761 DEBUG [master/bd6592c37bb2:0:becomeActiveMaster {}] regionserver.HRegion(1093): writing seq id for 1595e783b53d99cd5eef43b6debb2682 2024-11-24T22:13:44,766 DEBUG [master/bd6592c37bb2:0:becomeActiveMaster {}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-24T22:13:44,767 INFO [master/bd6592c37bb2:0:becomeActiveMaster {}] regionserver.HRegion(1114): Opened 1595e783b53d99cd5eef43b6debb2682; next sequenceid=2; ConstantSizeRegionSplitPolicy{desiredMaxFileSize=65861655, jitterRate=-0.01858486235141754}, FlushLargeStoresPolicy{flushSizeLowerBound=33554432} 2024-11-24T22:13:44,773 DEBUG [master/bd6592c37bb2:0:becomeActiveMaster {}] regionserver.HRegion(1006): Region open journal for 1595e783b53d99cd5eef43b6debb2682: Writing region info on filesystem at 1732486424677Initializing all the Stores at 1732486424680 (+3 ms)Instantiating store for column family {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1732486424680Instantiating store for column family {NAME => 'proc', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732486424681 (+1 ms)Instantiating store for column family {NAME => 'rs', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732486424681Instantiating store for column family {NAME => 'state', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732486424681Cleaning up temporary data from old regions at 1732486424755 (+74 ms)Region opened successfully at 1732486424773 (+18 ms) 2024-11-24T22:13:44,774 INFO [master/bd6592c37bb2:0:becomeActiveMaster {}] region.MasterRegionFlusherAndCompactor(122): Constructor flushSize=134217728, flushPerChanges=1000000, flushIntervalMs=900000, compactMin=4 2024-11-24T22:13:44,815 DEBUG [master/bd6592c37bb2:0:becomeActiveMaster {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@2d556f6c, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=bd6592c37bb2/172.17.0.2:0 2024-11-24T22:13:44,851 INFO [master/bd6592c37bb2:0:becomeActiveMaster {}] master.HMaster(912): No meta location available on zookeeper, skip migrating... 2024-11-24T22:13:44,864 INFO [master/bd6592c37bb2:0:becomeActiveMaster {}] region.RegionProcedureStore(104): Starting the Region Procedure Store, number threads=5 2024-11-24T22:13:44,864 INFO [master/bd6592c37bb2:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(626): Starting 5 core workers (bigger of cpus/4 or 16) with max (burst) worker count=50 2024-11-24T22:13:44,867 INFO [master/bd6592c37bb2:0:becomeActiveMaster {}] region.RegionProcedureStore(255): Starting Region Procedure Store lease recovery... 2024-11-24T22:13:44,869 INFO [master/bd6592c37bb2:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(676): Recovered RegionProcedureStore lease in 1 msec 2024-11-24T22:13:44,874 INFO [master/bd6592c37bb2:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(690): Loaded RegionProcedureStore in 4 msec 2024-11-24T22:13:44,874 INFO [master/bd6592c37bb2:0:becomeActiveMaster {}] procedure2.RemoteProcedureDispatcher(96): Instantiated, coreThreads=3 (allowCoreThreadTimeOut=true), queueMaxSize=32, operationDelay=150 2024-11-24T22:13:44,900 INFO [master/bd6592c37bb2:0:becomeActiveMaster {}] master.RegionServerTracker(127): Upgrading RegionServerTracker to active master mode; 0 have existingServerCrashProcedures, 0 possibly 'live' servers, and 0 'splitting'. 2024-11-24T22:13:44,909 DEBUG [master/bd6592c37bb2:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:37701-0x1014c0ac8d60000, quorum=127.0.0.1:59994, baseZNode=/hbase Unable to get data of znode /hbase/balancer because node does not exist (not necessarily an error) 2024-11-24T22:13:44,911 DEBUG [master/bd6592c37bb2:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/balancer already deleted, retry=false 2024-11-24T22:13:44,914 INFO [master/bd6592c37bb2:0:becomeActiveMaster {}] normalizer.SimpleRegionNormalizer(163): Updated configuration for key 'hbase.normalizer.merge.min_region_size.mb' from 0 to 1 2024-11-24T22:13:44,915 DEBUG [master/bd6592c37bb2:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:37701-0x1014c0ac8d60000, quorum=127.0.0.1:59994, baseZNode=/hbase Unable to get data of znode /hbase/normalizer because node does not exist (not necessarily an error) 2024-11-24T22:13:44,918 DEBUG [master/bd6592c37bb2:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/normalizer already deleted, retry=false 2024-11-24T22:13:44,920 INFO [master/bd6592c37bb2:0:becomeActiveMaster {}] normalizer.RegionNormalizerWorker(137): Normalizer rate limit set to unlimited 2024-11-24T22:13:44,923 DEBUG [master/bd6592c37bb2:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:37701-0x1014c0ac8d60000, quorum=127.0.0.1:59994, baseZNode=/hbase Unable to get data of znode /hbase/switch/split because node does not exist (not necessarily an error) 2024-11-24T22:13:44,925 DEBUG [master/bd6592c37bb2:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/switch/split already deleted, retry=false 2024-11-24T22:13:44,926 DEBUG [master/bd6592c37bb2:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:37701-0x1014c0ac8d60000, quorum=127.0.0.1:59994, baseZNode=/hbase Unable to get data of znode /hbase/switch/merge because node does not exist (not necessarily an error) 2024-11-24T22:13:44,930 DEBUG [master/bd6592c37bb2:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/switch/merge already deleted, retry=false 2024-11-24T22:13:44,948 DEBUG [master/bd6592c37bb2:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:37701-0x1014c0ac8d60000, quorum=127.0.0.1:59994, baseZNode=/hbase Unable to get data of znode /hbase/snapshot-cleanup because node does not exist (not necessarily an error) 2024-11-24T22:13:44,950 DEBUG [master/bd6592c37bb2:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/snapshot-cleanup already deleted, retry=false 2024-11-24T22:13:44,954 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:37701-0x1014c0ac8d60000, quorum=127.0.0.1:59994, baseZNode=/hbase Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/hbase/running 2024-11-24T22:13:44,954 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:40087-0x1014c0ac8d60001, quorum=127.0.0.1:59994, baseZNode=/hbase Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/hbase/running 2024-11-24T22:13:44,955 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:37701-0x1014c0ac8d60000, quorum=127.0.0.1:59994, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-24T22:13:44,955 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:40087-0x1014c0ac8d60001, quorum=127.0.0.1:59994, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-24T22:13:44,957 INFO [master/bd6592c37bb2:0:becomeActiveMaster {}] master.HMaster(856): Active/primary master=bd6592c37bb2,37701,1732486422992, sessionid=0x1014c0ac8d60000, setting cluster-up flag (Was=false) 2024-11-24T22:13:44,971 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:40087-0x1014c0ac8d60001, quorum=127.0.0.1:59994, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-24T22:13:44,971 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:37701-0x1014c0ac8d60000, quorum=127.0.0.1:59994, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-24T22:13:44,978 DEBUG [master/bd6592c37bb2:0:becomeActiveMaster {}] procedure.ZKProcedureUtil(251): Clearing all znodes /hbase/flush-table-proc/acquired, /hbase/flush-table-proc/reached, /hbase/flush-table-proc/abort 2024-11-24T22:13:44,979 DEBUG [master/bd6592c37bb2:0:becomeActiveMaster {}] procedure.ZKProcedureCoordinator(245): Starting controller for procedure member=bd6592c37bb2,37701,1732486422992 2024-11-24T22:13:44,986 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:40087-0x1014c0ac8d60001, quorum=127.0.0.1:59994, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-24T22:13:44,986 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:37701-0x1014c0ac8d60000, quorum=127.0.0.1:59994, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-24T22:13:44,993 DEBUG [master/bd6592c37bb2:0:becomeActiveMaster {}] procedure.ZKProcedureUtil(251): Clearing all znodes /hbase/online-snapshot/acquired, /hbase/online-snapshot/reached, /hbase/online-snapshot/abort 2024-11-24T22:13:44,994 DEBUG [master/bd6592c37bb2:0:becomeActiveMaster {}] procedure.ZKProcedureCoordinator(245): Starting controller for procedure member=bd6592c37bb2,37701,1732486422992 2024-11-24T22:13:45,000 INFO [master/bd6592c37bb2:0:becomeActiveMaster {}] master.ServerManager(1185): No .lastflushedseqids found at hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/.lastflushedseqids will record last flushed sequence id for regions by regionserver report all over again 2024-11-24T22:13:45,012 INFO [RS:0;bd6592c37bb2:40087 {}] regionserver.HRegionServer(746): ClusterId : 3c461ac1-06fb-4739-8702-6368e685b03c 2024-11-24T22:13:45,014 DEBUG [RS:0;bd6592c37bb2:40087 {}] procedure.RegionServerProcedureManagerHost(43): Procedure flush-table-proc initializing 2024-11-24T22:13:45,019 DEBUG [RS:0;bd6592c37bb2:40087 {}] procedure.RegionServerProcedureManagerHost(45): Procedure flush-table-proc initialized 2024-11-24T22:13:45,019 DEBUG [RS:0;bd6592c37bb2:40087 {}] procedure.RegionServerProcedureManagerHost(43): Procedure online-snapshot initializing 2024-11-24T22:13:45,022 DEBUG [RS:0;bd6592c37bb2:40087 {}] procedure.RegionServerProcedureManagerHost(45): Procedure online-snapshot initialized 2024-11-24T22:13:45,023 DEBUG [RS:0;bd6592c37bb2:40087 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@532ddb9b, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=bd6592c37bb2/172.17.0.2:0 2024-11-24T22:13:45,038 DEBUG [RS:0;bd6592c37bb2:40087 {}] regionserver.ShutdownHook(81): Installed shutdown hook thread: Shutdownhook:RS:0;bd6592c37bb2:40087 2024-11-24T22:13:45,042 INFO [RS:0;bd6592c37bb2:40087 {}] regionserver.RegionServerCoprocessorHost(66): System coprocessor loading is enabled 2024-11-24T22:13:45,042 INFO [RS:0;bd6592c37bb2:40087 {}] regionserver.RegionServerCoprocessorHost(67): Table coprocessor loading is enabled 2024-11-24T22:13:45,042 DEBUG [RS:0;bd6592c37bb2:40087 {}] regionserver.HRegionServer(832): About to register with Master. 2024-11-24T22:13:45,045 INFO [RS:0;bd6592c37bb2:40087 {}] regionserver.HRegionServer(2659): reportForDuty to master=bd6592c37bb2,37701,1732486422992 with port=40087, startcode=1732486423713 2024-11-24T22:13:45,060 DEBUG [RS:0;bd6592c37bb2:40087 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=RegionServerStatusService, sasl=false 2024-11-24T22:13:45,079 DEBUG [master/bd6592c37bb2:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(1139): Stored pid=1, state=RUNNABLE:INIT_META_WRITE_FS_LAYOUT, hasLock=false; InitMetaProcedure table=hbase:meta 2024-11-24T22:13:45,092 INFO [master/bd6592c37bb2:0:becomeActiveMaster {}] balancer.BaseLoadBalancer(416): slop=0.2 2024-11-24T22:13:45,094 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:35451, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins.hfs.0 (auth:SIMPLE), service=RegionServerStatusService 2024-11-24T22:13:45,100 DEBUG [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=37701 {}] ipc.MetricsHBaseServer(152): Unknown exception type org.apache.hadoop.hbase.ipc.ServerNotRunningYetException: Server is not running yet at org.apache.hadoop.hbase.master.HMaster.checkServiceStarted(HMaster.java:3334) ~[classes/:?] at org.apache.hadoop.hbase.master.MasterRpcServices.regionServerStartup(MasterRpcServices.java:667) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16714) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-24T22:13:45,102 INFO [master/bd6592c37bb2:0:becomeActiveMaster {}] balancer.StochasticLoadBalancer(272): Loaded config; maxSteps=1000000, runMaxSteps=false, stepsPerRegion=800, maxRunningTime=30000, isByTable=false, CostFunctions=[RegionCountSkewCostFunction, PrimaryRegionCountSkewCostFunction, MoveCostFunction, ServerLocalityCostFunction, RackLocalityCostFunction, TableSkewCostFunction, RegionReplicaHostCostFunction, RegionReplicaRackCostFunction, ReadRequestCostFunction, CPRequestCostFunction, WriteRequestCostFunction, MemStoreSizeCostFunction, StoreFileCostFunction] , sum of multiplier of cost functions = 0.0 etc. 2024-11-24T22:13:45,108 DEBUG [master/bd6592c37bb2:0:becomeActiveMaster {}] balancer.RegionHDFSBlockLocationFinder(133): Skipping locality-based refresh due to oldStatus=null, newStatus=Master: bd6592c37bb2,37701,1732486422992 Number of backup masters: 0 Number of live region servers: 0 Number of dead region servers: 0 Number of unknown region servers: 0 Average load: 0.0 Number of requests: 0 Number of regions: 0 Number of regions in transition: 0 2024-11-24T22:13:45,117 DEBUG [master/bd6592c37bb2:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_OPEN_REGION-master/bd6592c37bb2:0, corePoolSize=5, maxPoolSize=5 2024-11-24T22:13:45,117 DEBUG [master/bd6592c37bb2:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_CLOSE_REGION-master/bd6592c37bb2:0, corePoolSize=5, maxPoolSize=5 2024-11-24T22:13:45,118 DEBUG [master/bd6592c37bb2:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_SERVER_OPERATIONS-master/bd6592c37bb2:0, corePoolSize=5, maxPoolSize=5 2024-11-24T22:13:45,118 DEBUG [master/bd6592c37bb2:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_META_SERVER_OPERATIONS-master/bd6592c37bb2:0, corePoolSize=5, maxPoolSize=5 2024-11-24T22:13:45,118 DEBUG [master/bd6592c37bb2:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=M_LOG_REPLAY_OPS-master/bd6592c37bb2:0, corePoolSize=10, maxPoolSize=10 2024-11-24T22:13:45,118 DEBUG [master/bd6592c37bb2:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_SNAPSHOT_OPERATIONS-master/bd6592c37bb2:0, corePoolSize=1, maxPoolSize=1 2024-11-24T22:13:45,119 DEBUG [master/bd6592c37bb2:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_MERGE_OPERATIONS-master/bd6592c37bb2:0, corePoolSize=2, maxPoolSize=2 2024-11-24T22:13:45,119 DEBUG [master/bd6592c37bb2:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_TABLE_OPERATIONS-master/bd6592c37bb2:0, corePoolSize=1, maxPoolSize=1 2024-11-24T22:13:45,121 INFO [master/bd6592c37bb2:0:becomeActiveMaster {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.procedure2.CompletedProcedureCleaner; timeout=30000, timestamp=1732486455120 2024-11-24T22:13:45,122 INFO [master/bd6592c37bb2:0:becomeActiveMaster {}] cleaner.DirScanPool(74): log_cleaner Cleaner pool size is 1 2024-11-24T22:13:45,123 INFO [master/bd6592c37bb2:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveLogCleaner 2024-11-24T22:13:45,124 DEBUG [PEWorker-1 {}] procedure.InitMetaProcedure(97): Execute pid=1, state=RUNNABLE:INIT_META_WRITE_FS_LAYOUT, hasLock=true; InitMetaProcedure table=hbase:meta 2024-11-24T22:13:45,125 INFO [PEWorker-1 {}] procedure.InitMetaProcedure(76): BOOTSTRAP: creating hbase:meta region 2024-11-24T22:13:45,127 INFO [master/bd6592c37bb2:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.replication.master.ReplicationLogCleaner 2024-11-24T22:13:45,127 INFO [master/bd6592c37bb2:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveMasterLocalStoreWALCleaner 2024-11-24T22:13:45,127 INFO [master/bd6592c37bb2:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveProcedureWALCleaner 2024-11-24T22:13:45,128 INFO [master/bd6592c37bb2:0:becomeActiveMaster {}] cleaner.LogCleaner(148): Creating 1 old WALs cleaner threads 2024-11-24T22:13:45,131 DEBUG [PEWorker-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T22:13:45,130 INFO [master/bd6592c37bb2:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=LogsCleaner, period=600000, unit=MILLISECONDS is enabled. 2024-11-24T22:13:45,131 INFO [PEWorker-1 {}] util.FSTableDescriptors(156): Creating new hbase:meta table descriptor 'hbase:meta', {TABLE_ATTRIBUTES => {IS_META => 'true', coprocessor$1 => '|org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint|536870911|', METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'ns', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'rep_barrier', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '2147483647', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'table', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} 2024-11-24T22:13:45,133 INFO [master/bd6592c37bb2:0:becomeActiveMaster {}] cleaner.DirScanPool(74): hfile_cleaner Cleaner pool size is 2 2024-11-24T22:13:45,134 DEBUG [RS:0;bd6592c37bb2:40087 {}] regionserver.HRegionServer(2683): Master is not running yet 2024-11-24T22:13:45,134 WARN [RS:0;bd6592c37bb2:40087 {}] regionserver.HRegionServer(841): reportForDuty failed; sleeping 100 ms and then retrying. 2024-11-24T22:13:45,134 INFO [master/bd6592c37bb2:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveMasterLocalStoreHFileCleaner 2024-11-24T22:13:45,134 INFO [master/bd6592c37bb2:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.HFileLinkCleaner 2024-11-24T22:13:45,136 INFO [master/bd6592c37bb2:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.snapshot.SnapshotHFileCleaner 2024-11-24T22:13:45,137 INFO [master/bd6592c37bb2:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveHFileCleaner 2024-11-24T22:13:45,140 DEBUG [master/bd6592c37bb2:0:becomeActiveMaster {}] cleaner.HFileCleaner(260): Starting for large file=Thread[master/bd6592c37bb2:0:becomeActiveMaster-HFileCleaner.large.0-1732486425138,5,FailOnTimeoutGroup] 2024-11-24T22:13:45,141 DEBUG [master/bd6592c37bb2:0:becomeActiveMaster {}] cleaner.HFileCleaner(275): Starting for small files=Thread[master/bd6592c37bb2:0:becomeActiveMaster-HFileCleaner.small.0-1732486425140,5,FailOnTimeoutGroup] 2024-11-24T22:13:45,141 INFO [master/bd6592c37bb2:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=HFileCleaner, period=600000, unit=MILLISECONDS is enabled. 2024-11-24T22:13:45,141 INFO [master/bd6592c37bb2:0:becomeActiveMaster {}] master.HMaster(1741): Reopening regions with very high storeFileRefCount is disabled. Provide threshold value > 0 for hbase.regions.recovery.store.file.ref.count to enable it. 2024-11-24T22:13:45,141 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45929 is added to blk_1073741831_1007 (size=1321) 2024-11-24T22:13:45,142 INFO [master/bd6592c37bb2:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=ReplicationBarrierCleaner, period=43200000, unit=MILLISECONDS is enabled. 2024-11-24T22:13:45,142 INFO [master/bd6592c37bb2:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=SnapshotCleaner, period=1800000, unit=MILLISECONDS is enabled. 2024-11-24T22:13:45,143 INFO [PEWorker-1 {}] util.FSTableDescriptors(163): Updated hbase:meta table descriptor to hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/hbase/meta/.tabledesc/.tableinfo.0000000001.1321 2024-11-24T22:13:45,144 INFO [PEWorker-1 {}] regionserver.HRegion(7572): creating {ENCODED => 1588230740, NAME => 'hbase:meta,,1', STARTKEY => '', ENDKEY => ''}, tableDescriptor='hbase:meta', {TABLE_ATTRIBUTES => {IS_META => 'true', coprocessor$1 => '|org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint|536870911|', METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'ns', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'rep_barrier', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '2147483647', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'table', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, regionDir=hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc 2024-11-24T22:13:45,152 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45929 is added to blk_1073741832_1008 (size=32) 2024-11-24T22:13:45,235 INFO [RS:0;bd6592c37bb2:40087 {}] regionserver.HRegionServer(2659): reportForDuty to master=bd6592c37bb2,37701,1732486422992 with port=40087, startcode=1732486423713 2024-11-24T22:13:45,237 INFO [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=37701 {}] master.ServerManager(363): Checking decommissioned status of RegionServer bd6592c37bb2,40087,1732486423713 2024-11-24T22:13:45,240 INFO [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=37701 {}] master.ServerManager(517): Registering regionserver=bd6592c37bb2,40087,1732486423713 2024-11-24T22:13:45,248 DEBUG [RS:0;bd6592c37bb2:40087 {}] regionserver.HRegionServer(1440): Config from master: hbase.rootdir=hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc 2024-11-24T22:13:45,248 DEBUG [RS:0;bd6592c37bb2:40087 {}] regionserver.HRegionServer(1440): Config from master: fs.defaultFS=hdfs://localhost:43173 2024-11-24T22:13:45,249 DEBUG [RS:0;bd6592c37bb2:40087 {}] regionserver.HRegionServer(1440): Config from master: hbase.master.info.port=-1 2024-11-24T22:13:45,254 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:37701-0x1014c0ac8d60000, quorum=127.0.0.1:59994, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/rs 2024-11-24T22:13:45,255 DEBUG [RS:0;bd6592c37bb2:40087 {}] zookeeper.ZKUtil(111): regionserver:40087-0x1014c0ac8d60001, quorum=127.0.0.1:59994, baseZNode=/hbase Set watcher on existing znode=/hbase/rs/bd6592c37bb2,40087,1732486423713 2024-11-24T22:13:45,255 WARN [RS:0;bd6592c37bb2:40087 {}] hbase.ZNodeClearer(65): Environment variable HBASE_ZNODE_FILE not set; znodes will not be cleared on crash by start scripts (Longer MTTR!) 2024-11-24T22:13:45,255 INFO [RS:0;bd6592c37bb2:40087 {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-11-24T22:13:45,256 DEBUG [RS:0;bd6592c37bb2:40087 {}] regionserver.HRegionServer(1793): logDir=hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/WALs/bd6592c37bb2,40087,1732486423713 2024-11-24T22:13:45,258 INFO [RegionServerTracker-0 {}] master.RegionServerTracker(179): RegionServer ephemeral node created, adding [bd6592c37bb2,40087,1732486423713] 2024-11-24T22:13:45,284 INFO [RS:0;bd6592c37bb2:40087 {}] regionserver.MetricsRegionServerWrapperImpl(120): Computing regionserver metrics every 5000 milliseconds 2024-11-24T22:13:45,298 INFO [RS:0;bd6592c37bb2:40087 {}] regionserver.MemStoreFlusher(131): globalMemStoreLimit=880 M, globalMemStoreLimitLowMark=836 M, Offheap=false 2024-11-24T22:13:45,303 INFO [RS:0;bd6592c37bb2:40087 {}] throttle.PressureAwareCompactionThroughputController(131): Compaction throughput configurations, higher bound: 100.00 MB/second, lower bound 50.00 MB/second, off peak: unlimited, tuning period: 60000 ms 2024-11-24T22:13:45,303 INFO [RS:0;bd6592c37bb2:40087 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactionThroughputTuner, period=60000, unit=MILLISECONDS is enabled. 2024-11-24T22:13:45,304 INFO [RS:0;bd6592c37bb2:40087 {}] regionserver.HRegionServer$CompactionChecker(1680): CompactionChecker runs every PT1S 2024-11-24T22:13:45,309 INFO [RS:0;bd6592c37bb2:40087 {}] hbase.ExecutorStatusChore(48): ExecutorStatusChore runs every 1mins, 0sec 2024-11-24T22:13:45,311 INFO [RS:0;bd6592c37bb2:40087 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactedHFilesCleaner, period=120000, unit=MILLISECONDS is enabled. 2024-11-24T22:13:45,311 DEBUG [RS:0;bd6592c37bb2:40087 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_REGION-regionserver/bd6592c37bb2:0, corePoolSize=1, maxPoolSize=1 2024-11-24T22:13:45,311 DEBUG [RS:0;bd6592c37bb2:40087 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_META-regionserver/bd6592c37bb2:0, corePoolSize=1, maxPoolSize=1 2024-11-24T22:13:45,311 DEBUG [RS:0;bd6592c37bb2:40087 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_PRIORITY_REGION-regionserver/bd6592c37bb2:0, corePoolSize=1, maxPoolSize=1 2024-11-24T22:13:45,312 DEBUG [RS:0;bd6592c37bb2:40087 {}] executor.ExecutorService(95): Starting executor service name=RS_CLOSE_REGION-regionserver/bd6592c37bb2:0, corePoolSize=1, maxPoolSize=1 2024-11-24T22:13:45,312 DEBUG [RS:0;bd6592c37bb2:40087 {}] executor.ExecutorService(95): Starting executor service name=RS_CLOSE_META-regionserver/bd6592c37bb2:0, corePoolSize=1, maxPoolSize=1 2024-11-24T22:13:45,312 DEBUG [RS:0;bd6592c37bb2:40087 {}] executor.ExecutorService(95): Starting executor service name=RS_LOG_REPLAY_OPS-regionserver/bd6592c37bb2:0, corePoolSize=2, maxPoolSize=2 2024-11-24T22:13:45,312 DEBUG [RS:0;bd6592c37bb2:40087 {}] executor.ExecutorService(95): Starting executor service name=RS_COMPACTED_FILES_DISCHARGER-regionserver/bd6592c37bb2:0, corePoolSize=1, maxPoolSize=1 2024-11-24T22:13:45,312 DEBUG [RS:0;bd6592c37bb2:40087 {}] executor.ExecutorService(95): Starting executor service name=RS_REGION_REPLICA_FLUSH_OPS-regionserver/bd6592c37bb2:0, corePoolSize=1, maxPoolSize=1 2024-11-24T22:13:45,313 DEBUG [RS:0;bd6592c37bb2:40087 {}] executor.ExecutorService(95): Starting executor service name=RS_REFRESH_PEER-regionserver/bd6592c37bb2:0, corePoolSize=1, maxPoolSize=1 2024-11-24T22:13:45,313 DEBUG [RS:0;bd6592c37bb2:40087 {}] executor.ExecutorService(95): Starting executor service name=RS_REPLAY_SYNC_REPLICATION_WAL-regionserver/bd6592c37bb2:0, corePoolSize=1, maxPoolSize=1 2024-11-24T22:13:45,313 DEBUG [RS:0;bd6592c37bb2:40087 {}] executor.ExecutorService(95): Starting executor service name=RS_SWITCH_RPC_THROTTLE-regionserver/bd6592c37bb2:0, corePoolSize=1, maxPoolSize=1 2024-11-24T22:13:45,313 DEBUG [RS:0;bd6592c37bb2:40087 {}] executor.ExecutorService(95): Starting executor service name=RS_CLAIM_REPLICATION_QUEUE-regionserver/bd6592c37bb2:0, corePoolSize=1, maxPoolSize=1 2024-11-24T22:13:45,313 DEBUG [RS:0;bd6592c37bb2:40087 {}] executor.ExecutorService(95): Starting executor service name=RS_SNAPSHOT_OPERATIONS-regionserver/bd6592c37bb2:0, corePoolSize=3, maxPoolSize=3 2024-11-24T22:13:45,313 DEBUG [RS:0;bd6592c37bb2:40087 {}] executor.ExecutorService(95): Starting executor service name=RS_FLUSH_OPERATIONS-regionserver/bd6592c37bb2:0, corePoolSize=3, maxPoolSize=3 2024-11-24T22:13:45,314 INFO [RS:0;bd6592c37bb2:40087 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactionChecker, period=1000, unit=MILLISECONDS is enabled. 2024-11-24T22:13:45,314 INFO [RS:0;bd6592c37bb2:40087 {}] hbase.ChoreService(168): Chore ScheduledChore name=MemstoreFlusherChore, period=1000, unit=MILLISECONDS is enabled. 2024-11-24T22:13:45,314 INFO [RS:0;bd6592c37bb2:40087 {}] hbase.ChoreService(168): Chore ScheduledChore name=ExecutorStatusChore, period=60000, unit=MILLISECONDS is enabled. 2024-11-24T22:13:45,315 INFO [RS:0;bd6592c37bb2:40087 {}] hbase.ChoreService(168): Chore ScheduledChore name=nonceCleaner, period=360000, unit=MILLISECONDS is enabled. 2024-11-24T22:13:45,315 INFO [RS:0;bd6592c37bb2:40087 {}] hbase.ChoreService(168): Chore ScheduledChore name=BrokenStoreFileCleaner, period=21600000, unit=MILLISECONDS is enabled. 2024-11-24T22:13:45,315 INFO [RS:0;bd6592c37bb2:40087 {}] hbase.ChoreService(168): Chore ScheduledChore name=bd6592c37bb2,40087,1732486423713-MobFileCleanerChore, period=86400, unit=SECONDS is enabled. 2024-11-24T22:13:45,334 INFO [RS:0;bd6592c37bb2:40087 {}] regionserver.HeapMemoryManager(209): Starting, tuneOn=false 2024-11-24T22:13:45,336 INFO [RS:0;bd6592c37bb2:40087 {}] hbase.ChoreService(168): Chore ScheduledChore name=bd6592c37bb2,40087,1732486423713-HeapMemoryTunerChore, period=60000, unit=MILLISECONDS is enabled. 2024-11-24T22:13:45,337 INFO [RS:0;bd6592c37bb2:40087 {}] hbase.ChoreService(168): Chore ScheduledChore name=ReplicationSourceStatistics, period=300000, unit=MILLISECONDS is enabled. 2024-11-24T22:13:45,337 INFO [RS:0;bd6592c37bb2:40087 {}] regionserver.Replication(171): bd6592c37bb2,40087,1732486423713 started 2024-11-24T22:13:45,355 INFO [RS:0;bd6592c37bb2:40087 {}] hbase.ChoreService(168): Chore ScheduledChore name=ReplicationSinkStatistics, period=300000, unit=MILLISECONDS is enabled. 2024-11-24T22:13:45,355 INFO [RS:0;bd6592c37bb2:40087 {}] regionserver.HRegionServer(1482): Serving as bd6592c37bb2,40087,1732486423713, RpcServer on bd6592c37bb2/172.17.0.2:40087, sessionid=0x1014c0ac8d60001 2024-11-24T22:13:45,356 DEBUG [RS:0;bd6592c37bb2:40087 {}] procedure.RegionServerProcedureManagerHost(51): Procedure flush-table-proc starting 2024-11-24T22:13:45,356 DEBUG [RS:0;bd6592c37bb2:40087 {}] flush.RegionServerFlushTableProcedureManager(108): Start region server flush procedure manager bd6592c37bb2,40087,1732486423713 2024-11-24T22:13:45,357 DEBUG [RS:0;bd6592c37bb2:40087 {}] procedure.ZKProcedureMemberRpcs(357): Starting procedure member 'bd6592c37bb2,40087,1732486423713' 2024-11-24T22:13:45,357 DEBUG [RS:0;bd6592c37bb2:40087 {}] procedure.ZKProcedureMemberRpcs(134): Checking for aborted procedures on node: '/hbase/flush-table-proc/abort' 2024-11-24T22:13:45,358 DEBUG [RS:0;bd6592c37bb2:40087 {}] procedure.ZKProcedureMemberRpcs(154): Looking for new procedures under znode:'/hbase/flush-table-proc/acquired' 2024-11-24T22:13:45,358 DEBUG [RS:0;bd6592c37bb2:40087 {}] procedure.RegionServerProcedureManagerHost(53): Procedure flush-table-proc started 2024-11-24T22:13:45,359 DEBUG [RS:0;bd6592c37bb2:40087 {}] procedure.RegionServerProcedureManagerHost(51): Procedure online-snapshot starting 2024-11-24T22:13:45,359 DEBUG [RS:0;bd6592c37bb2:40087 {}] snapshot.RegionServerSnapshotManager(126): Start Snapshot Manager bd6592c37bb2,40087,1732486423713 2024-11-24T22:13:45,359 DEBUG [RS:0;bd6592c37bb2:40087 {}] procedure.ZKProcedureMemberRpcs(357): Starting procedure member 'bd6592c37bb2,40087,1732486423713' 2024-11-24T22:13:45,359 DEBUG [RS:0;bd6592c37bb2:40087 {}] procedure.ZKProcedureMemberRpcs(134): Checking for aborted procedures on node: '/hbase/online-snapshot/abort' 2024-11-24T22:13:45,359 DEBUG [RS:0;bd6592c37bb2:40087 {}] procedure.ZKProcedureMemberRpcs(154): Looking for new procedures under znode:'/hbase/online-snapshot/acquired' 2024-11-24T22:13:45,360 DEBUG [RS:0;bd6592c37bb2:40087 {}] procedure.RegionServerProcedureManagerHost(53): Procedure online-snapshot started 2024-11-24T22:13:45,360 INFO [RS:0;bd6592c37bb2:40087 {}] quotas.RegionServerRpcQuotaManager(64): Quota support disabled 2024-11-24T22:13:45,360 INFO [RS:0;bd6592c37bb2:40087 {}] quotas.RegionServerSpaceQuotaManager(80): Quota support disabled, not starting space quota manager. 2024-11-24T22:13:45,466 INFO [RS:0;bd6592c37bb2:40087 {}] monitor.StreamSlowMonitor(122): New stream slow monitor defaultMonitorName 2024-11-24T22:13:45,469 INFO [RS:0;bd6592c37bb2:40087 {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=bd6592c37bb2%2C40087%2C1732486423713, suffix=, logDir=hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/WALs/bd6592c37bb2,40087,1732486423713, archiveDir=hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/oldWALs, maxLogs=32 2024-11-24T22:13:45,486 DEBUG [RS:0;bd6592c37bb2:40087 {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/WALs/bd6592c37bb2,40087,1732486423713/bd6592c37bb2%2C40087%2C1732486423713.1732486425471, exclude list is [], retry=0 2024-11-24T22:13:45,490 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:45929,DS-37016b53-75fb-429c-b62c-a33fe5baf0d3,DISK] 2024-11-24T22:13:45,493 INFO [RS:0;bd6592c37bb2:40087 {}] wal.AbstractFSWAL(991): New WAL /user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/WALs/bd6592c37bb2,40087,1732486423713/bd6592c37bb2%2C40087%2C1732486423713.1732486425471 2024-11-24T22:13:45,494 DEBUG [RS:0;bd6592c37bb2:40087 {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:41971:41971)] 2024-11-24T22:13:45,554 DEBUG [PEWorker-1 {}] regionserver.HRegion(898): Instantiated hbase:meta,,1.1588230740; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-24T22:13:45,557 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family info of region 1588230740 2024-11-24T22:13:45,559 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName info 2024-11-24T22:13:45,559 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T22:13:45,560 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/info, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-24T22:13:45,561 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family ns of region 1588230740 2024-11-24T22:13:45,563 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName ns 2024-11-24T22:13:45,563 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T22:13:45,564 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/ns, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-24T22:13:45,564 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family rep_barrier of region 1588230740 2024-11-24T22:13:45,566 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName rep_barrier 2024-11-24T22:13:45,566 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T22:13:45,567 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/rep_barrier, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-24T22:13:45,567 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family table of region 1588230740 2024-11-24T22:13:45,570 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName table 2024-11-24T22:13:45,570 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T22:13:45,571 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/table, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-24T22:13:45,571 DEBUG [PEWorker-1 {}] regionserver.HRegion(1038): replaying wal for 1588230740 2024-11-24T22:13:45,572 DEBUG [PEWorker-1 {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/hbase/meta/1588230740 2024-11-24T22:13:45,573 DEBUG [PEWorker-1 {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/hbase/meta/1588230740 2024-11-24T22:13:45,576 DEBUG [PEWorker-1 {}] regionserver.HRegion(1048): stopping wal replay for 1588230740 2024-11-24T22:13:45,576 DEBUG [PEWorker-1 {}] regionserver.HRegion(1060): Cleaning up temporary data for 1588230740 2024-11-24T22:13:45,577 DEBUG [PEWorker-1 {}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table hbase:meta descriptor;using region.getMemStoreFlushHeapSize/# of families (16.0 M)) instead. 2024-11-24T22:13:45,579 DEBUG [PEWorker-1 {}] regionserver.HRegion(1093): writing seq id for 1588230740 2024-11-24T22:13:45,583 DEBUG [PEWorker-1 {}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/hbase/meta/1588230740/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-24T22:13:45,584 INFO [PEWorker-1 {}] regionserver.HRegion(1114): Opened 1588230740; next sequenceid=2; ConstantSizeRegionSplitPolicy{desiredMaxFileSize=71748333, jitterRate=0.06913347542285919}, FlushLargeStoresPolicy{flushSizeLowerBound=16777216} 2024-11-24T22:13:45,587 DEBUG [PEWorker-1 {}] regionserver.HRegion(1006): Region open journal for 1588230740: Writing region info on filesystem at 1732486425555Initializing all the Stores at 1732486425556 (+1 ms)Instantiating store for column family {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1732486425557 (+1 ms)Instantiating store for column family {NAME => 'ns', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1732486425557Instantiating store for column family {NAME => 'rep_barrier', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '2147483647', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732486425557Instantiating store for column family {NAME => 'table', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1732486425557Cleaning up temporary data from old regions at 1732486425576 (+19 ms)Region opened successfully at 1732486425587 (+11 ms) 2024-11-24T22:13:45,587 DEBUG [PEWorker-1 {}] regionserver.HRegion(1722): Closing 1588230740, disabling compactions & flushes 2024-11-24T22:13:45,587 INFO [PEWorker-1 {}] regionserver.HRegion(1755): Closing region hbase:meta,,1.1588230740 2024-11-24T22:13:45,587 DEBUG [PEWorker-1 {}] regionserver.HRegion(1776): Time limited wait for close lock on hbase:meta,,1.1588230740 2024-11-24T22:13:45,587 DEBUG [PEWorker-1 {}] regionserver.HRegion(1843): Acquired close lock on hbase:meta,,1.1588230740 after waiting 0 ms 2024-11-24T22:13:45,587 DEBUG [PEWorker-1 {}] regionserver.HRegion(1853): Updates disabled for region hbase:meta,,1.1588230740 2024-11-24T22:13:45,589 INFO [PEWorker-1 {}] regionserver.HRegion(1973): Closed hbase:meta,,1.1588230740 2024-11-24T22:13:45,589 DEBUG [PEWorker-1 {}] regionserver.HRegion(1676): Region close journal for 1588230740: Waiting for close lock at 1732486425587Disabling compacts and flushes for region at 1732486425587Disabling writes for close at 1732486425587Writing region close event to WAL at 1732486425588 (+1 ms)Closed at 1732486425589 (+1 ms) 2024-11-24T22:13:45,593 DEBUG [PEWorker-1 {}] procedure.InitMetaProcedure(97): Execute pid=1, state=RUNNABLE:INIT_META_ASSIGN_META, hasLock=true; InitMetaProcedure table=hbase:meta 2024-11-24T22:13:45,593 INFO [PEWorker-1 {}] procedure.InitMetaProcedure(108): Going to assign meta 2024-11-24T22:13:45,600 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=2, ppid=1, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN}] 2024-11-24T22:13:45,608 INFO [PEWorker-2 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=2, ppid=1, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN 2024-11-24T22:13:45,611 INFO [PEWorker-2 {}] assignment.TransitRegionStateProcedure(269): Starting pid=2, ppid=1, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN; state=OFFLINE, location=null; forceNewPlan=false, retain=false 2024-11-24T22:13:45,765 DEBUG [bd6592c37bb2:37701 {}] assignment.AssignmentManager(2472): Processing assignQueue; systemServersCount=1, allServersCount=1 2024-11-24T22:13:45,780 INFO [PEWorker-3 {}] assignment.RegionStateStore(223): pid=2 updating hbase:meta row=1588230740, regionState=OPENING, regionLocation=bd6592c37bb2,40087,1732486423713 2024-11-24T22:13:45,788 INFO [PEWorker-3 {}] zookeeper.MetaTableLocator(171): Setting hbase:meta replicaId=0 location in ZooKeeper as bd6592c37bb2,40087,1732486423713, state=OPENING 2024-11-24T22:13:45,796 DEBUG [PEWorker-3 {}] zookeeper.MetaTableLocator(183): hbase:meta region location doesn't exist, create it 2024-11-24T22:13:45,798 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:37701-0x1014c0ac8d60000, quorum=127.0.0.1:59994, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-24T22:13:45,799 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-24T22:13:45,801 DEBUG [PEWorker-3 {}] procedure2.ProcedureFutureUtil(75): The future has completed while adding callback, give up suspending procedure pid=2, ppid=1, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN 2024-11-24T22:13:45,801 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:40087-0x1014c0ac8d60001, quorum=127.0.0.1:59994, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-24T22:13:45,802 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-24T22:13:45,803 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=3, ppid=2, state=RUNNABLE, hasLock=false; OpenRegionProcedure 1588230740, server=bd6592c37bb2,40087,1732486423713}] 2024-11-24T22:13:45,985 DEBUG [RSProcedureDispatcher-pool-0 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=AdminService, sasl=false 2024-11-24T22:13:46,000 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:50017, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=AdminService 2024-11-24T22:13:46,011 INFO [RS_OPEN_META-regionserver/bd6592c37bb2:0-0 {event_type=M_RS_OPEN_META, pid=3}] handler.AssignRegionHandler(132): Open hbase:meta,,1.1588230740 2024-11-24T22:13:46,011 INFO [RS_OPEN_META-regionserver/bd6592c37bb2:0-0 {event_type=M_RS_OPEN_META, pid=3}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-11-24T22:13:46,012 INFO [RS_OPEN_META-regionserver/bd6592c37bb2:0-0 {event_type=M_RS_OPEN_META, pid=3}] monitor.StreamSlowMonitor(122): New stream slow monitor .meta 2024-11-24T22:13:46,016 INFO [RS_OPEN_META-regionserver/bd6592c37bb2:0-0 {event_type=M_RS_OPEN_META, pid=3}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=bd6592c37bb2%2C40087%2C1732486423713.meta, suffix=.meta, logDir=hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/WALs/bd6592c37bb2,40087,1732486423713, archiveDir=hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/oldWALs, maxLogs=32 2024-11-24T22:13:46,038 DEBUG [RS_OPEN_META-regionserver/bd6592c37bb2:0-0 {event_type=M_RS_OPEN_META, pid=3}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/WALs/bd6592c37bb2,40087,1732486423713/bd6592c37bb2%2C40087%2C1732486423713.meta.1732486426018.meta, exclude list is [], retry=0 2024-11-24T22:13:46,042 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:45929,DS-37016b53-75fb-429c-b62c-a33fe5baf0d3,DISK] 2024-11-24T22:13:46,046 INFO [RS_OPEN_META-regionserver/bd6592c37bb2:0-0 {event_type=M_RS_OPEN_META, pid=3}] wal.AbstractFSWAL(991): New WAL /user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/WALs/bd6592c37bb2,40087,1732486423713/bd6592c37bb2%2C40087%2C1732486423713.meta.1732486426018.meta 2024-11-24T22:13:46,047 DEBUG [RS_OPEN_META-regionserver/bd6592c37bb2:0-0 {event_type=M_RS_OPEN_META, pid=3}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:41971:41971)] 2024-11-24T22:13:46,047 DEBUG [RS_OPEN_META-regionserver/bd6592c37bb2:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(7752): Opening region: {ENCODED => 1588230740, NAME => 'hbase:meta,,1', STARTKEY => '', ENDKEY => ''} 2024-11-24T22:13:46,050 DEBUG [RS_OPEN_META-regionserver/bd6592c37bb2:0-0 {event_type=M_RS_OPEN_META, pid=3}] coprocessor.CoprocessorHost(215): Loading coprocessor class org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint with path null and priority 536870911 2024-11-24T22:13:46,052 DEBUG [RS_OPEN_META-regionserver/bd6592c37bb2:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(8280): Registered coprocessor service: region=hbase:meta,,1 service=MultiRowMutationService 2024-11-24T22:13:46,058 INFO [RS_OPEN_META-regionserver/bd6592c37bb2:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.RegionCoprocessorHost(434): Loaded coprocessor org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint from HTD of hbase:meta successfully. 2024-11-24T22:13:46,063 DEBUG [RS_OPEN_META-regionserver/bd6592c37bb2:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table meta 1588230740 2024-11-24T22:13:46,064 DEBUG [RS_OPEN_META-regionserver/bd6592c37bb2:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(898): Instantiated hbase:meta,,1.1588230740; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-24T22:13:46,064 DEBUG [RS_OPEN_META-regionserver/bd6592c37bb2:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(7794): checking encryption for 1588230740 2024-11-24T22:13:46,064 DEBUG [RS_OPEN_META-regionserver/bd6592c37bb2:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(7797): checking classloading for 1588230740 2024-11-24T22:13:46,068 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family info of region 1588230740 2024-11-24T22:13:46,070 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName info 2024-11-24T22:13:46,070 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T22:13:46,071 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/info, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-24T22:13:46,072 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family ns of region 1588230740 2024-11-24T22:13:46,074 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName ns 2024-11-24T22:13:46,074 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T22:13:46,075 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/ns, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-24T22:13:46,075 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family rep_barrier of region 1588230740 2024-11-24T22:13:46,077 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName rep_barrier 2024-11-24T22:13:46,077 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T22:13:46,078 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/rep_barrier, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-24T22:13:46,078 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family table of region 1588230740 2024-11-24T22:13:46,080 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName table 2024-11-24T22:13:46,080 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T22:13:46,081 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/table, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-24T22:13:46,082 DEBUG [RS_OPEN_META-regionserver/bd6592c37bb2:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1038): replaying wal for 1588230740 2024-11-24T22:13:46,083 DEBUG [RS_OPEN_META-regionserver/bd6592c37bb2:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/hbase/meta/1588230740 2024-11-24T22:13:46,086 DEBUG [RS_OPEN_META-regionserver/bd6592c37bb2:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/hbase/meta/1588230740 2024-11-24T22:13:46,090 DEBUG [RS_OPEN_META-regionserver/bd6592c37bb2:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1048): stopping wal replay for 1588230740 2024-11-24T22:13:46,090 DEBUG [RS_OPEN_META-regionserver/bd6592c37bb2:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1060): Cleaning up temporary data for 1588230740 2024-11-24T22:13:46,091 DEBUG [RS_OPEN_META-regionserver/bd6592c37bb2:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table hbase:meta descriptor;using region.getMemStoreFlushHeapSize/# of families (16.0 M)) instead. 2024-11-24T22:13:46,094 DEBUG [RS_OPEN_META-regionserver/bd6592c37bb2:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1093): writing seq id for 1588230740 2024-11-24T22:13:46,096 INFO [RS_OPEN_META-regionserver/bd6592c37bb2:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1114): Opened 1588230740; next sequenceid=2; ConstantSizeRegionSplitPolicy{desiredMaxFileSize=59651855, jitterRate=-0.11111809313297272}, FlushLargeStoresPolicy{flushSizeLowerBound=16777216} 2024-11-24T22:13:46,096 DEBUG [RS_OPEN_META-regionserver/bd6592c37bb2:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 1588230740 2024-11-24T22:13:46,099 DEBUG [RS_OPEN_META-regionserver/bd6592c37bb2:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1006): Region open journal for 1588230740: Running coprocessor pre-open hook at 1732486426065Writing region info on filesystem at 1732486426065Initializing all the Stores at 1732486426067 (+2 ms)Instantiating store for column family {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1732486426067Instantiating store for column family {NAME => 'ns', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1732486426068 (+1 ms)Instantiating store for column family {NAME => 'rep_barrier', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '2147483647', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732486426068Instantiating store for column family {NAME => 'table', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1732486426068Cleaning up temporary data from old regions at 1732486426090 (+22 ms)Running coprocessor post-open hooks at 1732486426096 (+6 ms)Region opened successfully at 1732486426099 (+3 ms) 2024-11-24T22:13:46,109 INFO [RS_OPEN_META-regionserver/bd6592c37bb2:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegionServer(2236): Post open deploy tasks for hbase:meta,,1.1588230740, pid=3, masterSystemTime=1732486425972 2024-11-24T22:13:46,123 DEBUG [RS_OPEN_META-regionserver/bd6592c37bb2:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegionServer(2266): Finished post open deploy task for hbase:meta,,1.1588230740 2024-11-24T22:13:46,123 INFO [RS_OPEN_META-regionserver/bd6592c37bb2:0-0 {event_type=M_RS_OPEN_META, pid=3}] handler.AssignRegionHandler(153): Opened hbase:meta,,1.1588230740 2024-11-24T22:13:46,125 INFO [PEWorker-5 {}] assignment.RegionStateStore(223): pid=2 updating hbase:meta row=1588230740, regionState=OPEN, openSeqNum=2, regionLocation=bd6592c37bb2,40087,1732486423713 2024-11-24T22:13:46,129 INFO [PEWorker-5 {}] zookeeper.MetaTableLocator(171): Setting hbase:meta replicaId=0 location in ZooKeeper as bd6592c37bb2,40087,1732486423713, state=OPEN 2024-11-24T22:13:46,136 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:40087-0x1014c0ac8d60001, quorum=127.0.0.1:59994, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-24T22:13:46,137 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-24T22:13:46,139 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:37701-0x1014c0ac8d60000, quorum=127.0.0.1:59994, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-24T22:13:46,139 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-24T22:13:46,141 DEBUG [PEWorker-5 {}] procedure2.ProcedureFutureUtil(75): The future has completed while adding callback, give up suspending procedure pid=3, ppid=2, state=RUNNABLE, hasLock=true; OpenRegionProcedure 1588230740, server=bd6592c37bb2,40087,1732486423713 2024-11-24T22:13:46,146 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=3, resume processing ppid=2 2024-11-24T22:13:46,147 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=3, ppid=2, state=SUCCESS, hasLock=false; OpenRegionProcedure 1588230740, server=bd6592c37bb2,40087,1732486423713 in 338 msec 2024-11-24T22:13:46,153 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=2, resume processing ppid=1 2024-11-24T22:13:46,153 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=2, ppid=1, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN in 548 msec 2024-11-24T22:13:46,155 DEBUG [PEWorker-2 {}] procedure.InitMetaProcedure(97): Execute pid=1, state=RUNNABLE:INIT_META_CREATE_NAMESPACES, hasLock=true; InitMetaProcedure table=hbase:meta 2024-11-24T22:13:46,155 INFO [PEWorker-2 {}] procedure.InitMetaProcedure(114): Going to create {NAME => 'default'} and {NAME => 'hbase'} namespaces 2024-11-24T22:13:46,177 DEBUG [PEWorker-2 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-24T22:13:46,179 DEBUG [PEWorker-2 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=bd6592c37bb2,40087,1732486423713, seqNum=-1] 2024-11-24T22:13:46,207 DEBUG [PEWorker-2 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-24T22:13:46,210 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:33575, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-24T22:13:46,233 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=1, state=SUCCESS, hasLock=false; InitMetaProcedure table=hbase:meta in 1.2030 sec 2024-11-24T22:13:46,233 INFO [master/bd6592c37bb2:0:becomeActiveMaster {}] master.HMaster(1123): Wait for region servers to report in: status=status unset, state=RUNNING, startTime=1732486426233, completionTime=-1 2024-11-24T22:13:46,236 INFO [master/bd6592c37bb2:0:becomeActiveMaster {}] master.ServerManager(903): Finished waiting on RegionServer count=1; waited=0ms, expected min=1 server(s), max=1 server(s), master is running 2024-11-24T22:13:46,237 DEBUG [master/bd6592c37bb2:0:becomeActiveMaster {}] assignment.AssignmentManager(1764): Joining cluster... 2024-11-24T22:13:46,264 INFO [master/bd6592c37bb2:0:becomeActiveMaster {}] assignment.AssignmentManager(1776): Number of RegionServers=1 2024-11-24T22:13:46,264 INFO [master/bd6592c37bb2:0:becomeActiveMaster {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$RegionInTransitionChore; timeout=60000, timestamp=1732486486264 2024-11-24T22:13:46,264 INFO [master/bd6592c37bb2:0:becomeActiveMaster {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$DeadServerMetricRegionChore; timeout=120000, timestamp=1732486546264 2024-11-24T22:13:46,264 INFO [master/bd6592c37bb2:0:becomeActiveMaster {}] assignment.AssignmentManager(1783): Joined the cluster in 27 msec 2024-11-24T22:13:46,267 INFO [master/bd6592c37bb2:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=bd6592c37bb2,37701,1732486422992-ClusterStatusChore, period=60000, unit=MILLISECONDS is enabled. 2024-11-24T22:13:46,267 INFO [master/bd6592c37bb2:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=bd6592c37bb2,37701,1732486422992-BalancerChore, period=300000, unit=MILLISECONDS is enabled. 2024-11-24T22:13:46,267 INFO [master/bd6592c37bb2:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=bd6592c37bb2,37701,1732486422992-RegionNormalizerChore, period=300000, unit=MILLISECONDS is enabled. 2024-11-24T22:13:46,269 INFO [master/bd6592c37bb2:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=CatalogJanitor-bd6592c37bb2:37701, period=300000, unit=MILLISECONDS is enabled. 2024-11-24T22:13:46,269 INFO [master/bd6592c37bb2:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=HbckChore-, period=3600000, unit=MILLISECONDS is enabled. 2024-11-24T22:13:46,270 INFO [master/bd6592c37bb2:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=FlushedSequenceIdFlusher, period=10800000, unit=MILLISECONDS is enabled. 2024-11-24T22:13:46,275 DEBUG [master/bd6592c37bb2:0.Chore.1 {}] janitor.CatalogJanitor(180): 2024-11-24T22:13:46,297 INFO [master/bd6592c37bb2:0:becomeActiveMaster {}] master.HMaster(1239): Master has completed initialization 2.439sec 2024-11-24T22:13:46,298 INFO [master/bd6592c37bb2:0:becomeActiveMaster {}] quotas.MasterQuotaManager(97): Quota support disabled 2024-11-24T22:13:46,299 INFO [master/bd6592c37bb2:0:becomeActiveMaster {}] slowlog.SlowLogMasterService(57): Slow/Large requests logging to system table hbase:slowlog is disabled. Quitting. 2024-11-24T22:13:46,300 INFO [master/bd6592c37bb2:0:becomeActiveMaster {}] waleventtracker.WALEventTrackerTableCreator(75): wal event tracker requests logging to table REPLICATION.WALEVENTTRACKER is disabled. Quitting. 2024-11-24T22:13:46,301 INFO [master/bd6592c37bb2:0:becomeActiveMaster {}] master.ReplicationSinkTrackerTableCreator(90): replication sink tracker requests logging to table REPLICATION.SINK_TRACKER is disabled. Quitting. 2024-11-24T22:13:46,301 INFO [master/bd6592c37bb2:0:becomeActiveMaster {}] zookeeper.ZKWatcher(271): not a secure deployment, proceeding 2024-11-24T22:13:46,302 INFO [master/bd6592c37bb2:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=bd6592c37bb2,37701,1732486422992-MobFileCleanerChore, period=86400, unit=SECONDS is enabled. 2024-11-24T22:13:46,302 INFO [master/bd6592c37bb2:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=bd6592c37bb2,37701,1732486422992-MobFileCompactionChore, period=604800, unit=SECONDS is enabled. 2024-11-24T22:13:46,310 DEBUG [master/bd6592c37bb2:0:becomeActiveMaster {}] master.HMaster(1374): Balancer post startup initialization complete, took 0 seconds 2024-11-24T22:13:46,311 INFO [master/bd6592c37bb2:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=RollingUpgradeChore, period=10, unit=SECONDS is enabled. 2024-11-24T22:13:46,312 INFO [master/bd6592c37bb2:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=bd6592c37bb2,37701,1732486422992-OldWALsDirSizeChore, period=300000, unit=MILLISECONDS is enabled. 2024-11-24T22:13:46,322 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@343aff6a, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-24T22:13:46,325 DEBUG [Time-limited test {}] nio.NioEventLoop(110): -Dio.netty.noKeySetOptimization: false 2024-11-24T22:13:46,325 DEBUG [Time-limited test {}] nio.NioEventLoop(111): -Dio.netty.selectorAutoRebuildThreshold: 512 2024-11-24T22:13:46,328 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request bd6592c37bb2,37701,-1 for getting cluster id 2024-11-24T22:13:46,351 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-24T22:13:46,360 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '3c461ac1-06fb-4739-8702-6368e685b03c' 2024-11-24T22:13:46,362 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-24T22:13:46,363 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "3c461ac1-06fb-4739-8702-6368e685b03c" 2024-11-24T22:13:46,365 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@519651b8, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-24T22:13:46,365 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [bd6592c37bb2,37701,-1] 2024-11-24T22:13:46,368 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-24T22:13:46,370 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-24T22:13:46,371 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:46876, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-24T22:13:46,377 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@32372268, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-24T22:13:46,378 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-24T22:13:46,384 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=bd6592c37bb2,40087,1732486423713, seqNum=-1] 2024-11-24T22:13:46,385 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-24T22:13:46,387 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:36508, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-24T22:13:46,407 INFO [Time-limited test {}] hbase.HBaseTestingUtil(877): Minicluster is up; activeMaster=bd6592c37bb2,37701,1732486422992 2024-11-24T22:13:46,419 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: TestAcidGuaranteesWithEagerPolicy#testMixedAtomicity Thread=211, OpenFileDescriptor=441, MaxFileDescriptor=1048576, SystemLoadAverage=301, ProcessCount=11, AvailableMemoryMB=6857 2024-11-24T22:13:46,427 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching master stub from registry 2024-11-24T22:13:46,432 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncConnectionImpl(321): The fetched master address is bd6592c37bb2,37701,1732486422992 2024-11-24T22:13:46,435 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.ConnectionUtils(555): The fetched master stub is org.apache.hadoop.hbase.shaded.protobuf.generated.MasterProtos$MasterService$Stub@3035477d 2024-11-24T22:13:46,435 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=MasterService, sasl=false 2024-11-24T22:13:46,438 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:46878, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=MasterService 2024-11-24T22:13:46,440 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37701 {}] util.TableDescriptorChecker(321): MEMSTORE_FLUSHSIZE for table descriptor or "hbase.hregion.memstore.flush.size" (131072) is too small, which might cause very frequent flushing. 2024-11-24T22:13:46,444 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37701 {}] master.HMaster$4(2454): Client=jenkins//172.17.0.2 create 'TestAcidGuarantees', {TABLE_ATTRIBUTES => {METADATA => {'hbase.hregion.compacting.memstore.type' => 'EAGER', 'hbase.hregion.memstore.mslab.enabled' => 'false', 'hbase.memstore.inmemoryflush.threshold.factor' => '0.9'}}}, {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-11-24T22:13:46,452 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37701 {}] procedure2.ProcedureExecutor(1139): Stored pid=4, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=false; CreateTableProcedure table=TestAcidGuarantees 2024-11-24T22:13:46,455 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_PRE_OPERATION 2024-11-24T22:13:46,458 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37701 {}] master.MasterRpcServices(787): Client=jenkins//172.17.0.2 procedure request for creating table: namespace: "default" qualifier: "TestAcidGuarantees" procId is: 4 2024-11-24T22:13:46,458 DEBUG [PEWorker-3 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T22:13:46,461 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_WRITE_FS_LAYOUT, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_WRITE_FS_LAYOUT 2024-11-24T22:13:46,463 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37701 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=4 2024-11-24T22:13:46,473 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45929 is added to blk_1073741835_1011 (size=1061) 2024-11-24T22:13:46,571 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37701 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=4 2024-11-24T22:13:46,781 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37701 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=4 2024-11-24T22:13:46,877 INFO [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(7572): creating {ENCODED => f0252adfc27b54a76a05585f3f45bf90, NAME => 'TestAcidGuarantees,,1732486426439.f0252adfc27b54a76a05585f3f45bf90.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='TestAcidGuarantees', {TABLE_ATTRIBUTES => {METADATA => {'hbase.hregion.compacting.memstore.type' => 'EAGER', 'hbase.hregion.memstore.mslab.enabled' => 'false', 'hbase.memstore.inmemoryflush.threshold.factor' => '0.9', 'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc 2024-11-24T22:13:46,887 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45929 is added to blk_1073741836_1012 (size=53) 2024-11-24T22:13:47,092 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37701 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=4 2024-11-24T22:13:47,290 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(898): Instantiated TestAcidGuarantees,,1732486426439.f0252adfc27b54a76a05585f3f45bf90.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-24T22:13:47,290 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1722): Closing f0252adfc27b54a76a05585f3f45bf90, disabling compactions & flushes 2024-11-24T22:13:47,290 INFO [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1755): Closing region TestAcidGuarantees,,1732486426439.f0252adfc27b54a76a05585f3f45bf90. 2024-11-24T22:13:47,290 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1776): Time limited wait for close lock on TestAcidGuarantees,,1732486426439.f0252adfc27b54a76a05585f3f45bf90. 2024-11-24T22:13:47,290 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1843): Acquired close lock on TestAcidGuarantees,,1732486426439.f0252adfc27b54a76a05585f3f45bf90. after waiting 0 ms 2024-11-24T22:13:47,290 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1853): Updates disabled for region TestAcidGuarantees,,1732486426439.f0252adfc27b54a76a05585f3f45bf90. 2024-11-24T22:13:47,291 INFO [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1973): Closed TestAcidGuarantees,,1732486426439.f0252adfc27b54a76a05585f3f45bf90. 2024-11-24T22:13:47,291 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1676): Region close journal for f0252adfc27b54a76a05585f3f45bf90: Waiting for close lock at 1732486427290Disabling compacts and flushes for region at 1732486427290Disabling writes for close at 1732486427290Writing region close event to WAL at 1732486427290Closed at 1732486427290 2024-11-24T22:13:47,294 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_ADD_TO_META, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_ADD_TO_META 2024-11-24T22:13:47,299 DEBUG [PEWorker-3 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":2,"row":"TestAcidGuarantees,,1732486426439.f0252adfc27b54a76a05585f3f45bf90.","families":{"info":[{"qualifier":"regioninfo","vlen":52,"tag":[],"timestamp":"1732486427294"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1732486427294"}]},"ts":"1732486427294"} 2024-11-24T22:13:47,307 INFO [PEWorker-3 {}] hbase.MetaTableAccessor(832): Added 1 regions to meta. 2024-11-24T22:13:47,310 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_ASSIGN_REGIONS, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_ASSIGN_REGIONS 2024-11-24T22:13:47,313 DEBUG [PEWorker-3 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1732486427310"}]},"ts":"1732486427310"} 2024-11-24T22:13:47,318 INFO [PEWorker-3 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=ENABLING in hbase:meta 2024-11-24T22:13:47,321 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=5, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=f0252adfc27b54a76a05585f3f45bf90, ASSIGN}] 2024-11-24T22:13:47,325 INFO [PEWorker-4 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=5, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=f0252adfc27b54a76a05585f3f45bf90, ASSIGN 2024-11-24T22:13:47,327 INFO [PEWorker-4 {}] assignment.TransitRegionStateProcedure(269): Starting pid=5, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=f0252adfc27b54a76a05585f3f45bf90, ASSIGN; state=OFFLINE, location=bd6592c37bb2,40087,1732486423713; forceNewPlan=false, retain=false 2024-11-24T22:13:47,478 INFO [PEWorker-5 {}] assignment.RegionStateStore(223): pid=5 updating hbase:meta row=f0252adfc27b54a76a05585f3f45bf90, regionState=OPENING, regionLocation=bd6592c37bb2,40087,1732486423713 2024-11-24T22:13:47,483 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=5, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=f0252adfc27b54a76a05585f3f45bf90, ASSIGN because future has completed 2024-11-24T22:13:47,484 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=6, ppid=5, state=RUNNABLE, hasLock=false; OpenRegionProcedure f0252adfc27b54a76a05585f3f45bf90, server=bd6592c37bb2,40087,1732486423713}] 2024-11-24T22:13:47,602 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37701 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=4 2024-11-24T22:13:47,644 INFO [RS_OPEN_REGION-regionserver/bd6592c37bb2:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] handler.AssignRegionHandler(132): Open TestAcidGuarantees,,1732486426439.f0252adfc27b54a76a05585f3f45bf90. 2024-11-24T22:13:47,645 DEBUG [RS_OPEN_REGION-regionserver/bd6592c37bb2:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(7752): Opening region: {ENCODED => f0252adfc27b54a76a05585f3f45bf90, NAME => 'TestAcidGuarantees,,1732486426439.f0252adfc27b54a76a05585f3f45bf90.', STARTKEY => '', ENDKEY => ''} 2024-11-24T22:13:47,645 DEBUG [RS_OPEN_REGION-regionserver/bd6592c37bb2:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table TestAcidGuarantees f0252adfc27b54a76a05585f3f45bf90 2024-11-24T22:13:47,645 DEBUG [RS_OPEN_REGION-regionserver/bd6592c37bb2:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(898): Instantiated TestAcidGuarantees,,1732486426439.f0252adfc27b54a76a05585f3f45bf90.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-24T22:13:47,645 DEBUG [RS_OPEN_REGION-regionserver/bd6592c37bb2:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(7794): checking encryption for f0252adfc27b54a76a05585f3f45bf90 2024-11-24T22:13:47,646 DEBUG [RS_OPEN_REGION-regionserver/bd6592c37bb2:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(7797): checking classloading for f0252adfc27b54a76a05585f3f45bf90 2024-11-24T22:13:47,648 INFO [StoreOpener-f0252adfc27b54a76a05585f3f45bf90-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family A of region f0252adfc27b54a76a05585f3f45bf90 2024-11-24T22:13:47,651 INFO [StoreOpener-f0252adfc27b54a76a05585f3f45bf90-1 {}] regionserver.CompactingMemStore(122): Store=A, in-memory flush size threshold=38.40 KB, immutable segments index type=ARRAY_MAP, compactor=EAGER, pipelineThreshold=2, compactionCellMax=10 2024-11-24T22:13:47,651 INFO [StoreOpener-f0252adfc27b54a76a05585f3f45bf90-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region f0252adfc27b54a76a05585f3f45bf90 columnFamilyName A 2024-11-24T22:13:47,651 DEBUG [StoreOpener-f0252adfc27b54a76a05585f3f45bf90-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T22:13:47,652 INFO [StoreOpener-f0252adfc27b54a76a05585f3f45bf90-1 {}] regionserver.HStore(327): Store=f0252adfc27b54a76a05585f3f45bf90/A, memstore type=CompactingMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-24T22:13:47,653 INFO [StoreOpener-f0252adfc27b54a76a05585f3f45bf90-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family B of region f0252adfc27b54a76a05585f3f45bf90 2024-11-24T22:13:47,655 INFO [StoreOpener-f0252adfc27b54a76a05585f3f45bf90-1 {}] regionserver.CompactingMemStore(122): Store=B, in-memory flush size threshold=38.40 KB, immutable segments index type=ARRAY_MAP, compactor=EAGER, pipelineThreshold=2, compactionCellMax=10 2024-11-24T22:13:47,655 INFO [StoreOpener-f0252adfc27b54a76a05585f3f45bf90-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region f0252adfc27b54a76a05585f3f45bf90 columnFamilyName B 2024-11-24T22:13:47,655 DEBUG [StoreOpener-f0252adfc27b54a76a05585f3f45bf90-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T22:13:47,656 INFO [StoreOpener-f0252adfc27b54a76a05585f3f45bf90-1 {}] regionserver.HStore(327): Store=f0252adfc27b54a76a05585f3f45bf90/B, memstore type=CompactingMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-24T22:13:47,656 INFO [StoreOpener-f0252adfc27b54a76a05585f3f45bf90-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family C of region f0252adfc27b54a76a05585f3f45bf90 2024-11-24T22:13:47,658 INFO [StoreOpener-f0252adfc27b54a76a05585f3f45bf90-1 {}] regionserver.CompactingMemStore(122): Store=C, in-memory flush size threshold=38.40 KB, immutable segments index type=ARRAY_MAP, compactor=EAGER, pipelineThreshold=2, compactionCellMax=10 2024-11-24T22:13:47,659 INFO [StoreOpener-f0252adfc27b54a76a05585f3f45bf90-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region f0252adfc27b54a76a05585f3f45bf90 columnFamilyName C 2024-11-24T22:13:47,659 DEBUG [StoreOpener-f0252adfc27b54a76a05585f3f45bf90-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T22:13:47,659 INFO [StoreOpener-f0252adfc27b54a76a05585f3f45bf90-1 {}] regionserver.HStore(327): Store=f0252adfc27b54a76a05585f3f45bf90/C, memstore type=CompactingMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-24T22:13:47,660 INFO [RS_OPEN_REGION-regionserver/bd6592c37bb2:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1212): Setting FlushNonSloppyStoresFirstPolicy for the region=TestAcidGuarantees,,1732486426439.f0252adfc27b54a76a05585f3f45bf90. 2024-11-24T22:13:47,660 DEBUG [RS_OPEN_REGION-regionserver/bd6592c37bb2:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1038): replaying wal for f0252adfc27b54a76a05585f3f45bf90 2024-11-24T22:13:47,661 DEBUG [RS_OPEN_REGION-regionserver/bd6592c37bb2:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/f0252adfc27b54a76a05585f3f45bf90 2024-11-24T22:13:47,662 DEBUG [RS_OPEN_REGION-regionserver/bd6592c37bb2:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/f0252adfc27b54a76a05585f3f45bf90 2024-11-24T22:13:47,664 DEBUG [RS_OPEN_REGION-regionserver/bd6592c37bb2:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1048): stopping wal replay for f0252adfc27b54a76a05585f3f45bf90 2024-11-24T22:13:47,664 DEBUG [RS_OPEN_REGION-regionserver/bd6592c37bb2:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1060): Cleaning up temporary data for f0252adfc27b54a76a05585f3f45bf90 2024-11-24T22:13:47,665 DEBUG [RS_OPEN_REGION-regionserver/bd6592c37bb2:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table TestAcidGuarantees descriptor;using region.getMemStoreFlushHeapSize/# of families (16.0 M)) instead. 2024-11-24T22:13:47,668 DEBUG [RS_OPEN_REGION-regionserver/bd6592c37bb2:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1093): writing seq id for f0252adfc27b54a76a05585f3f45bf90 2024-11-24T22:13:47,672 DEBUG [RS_OPEN_REGION-regionserver/bd6592c37bb2:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/f0252adfc27b54a76a05585f3f45bf90/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-24T22:13:47,672 INFO [RS_OPEN_REGION-regionserver/bd6592c37bb2:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1114): Opened f0252adfc27b54a76a05585f3f45bf90; next sequenceid=2; ConstantSizeRegionSplitPolicy{desiredMaxFileSize=67398886, jitterRate=0.004321664571762085}, FlushLargeStoresPolicy{flushSizeLowerBound=16777216} 2024-11-24T22:13:47,672 DEBUG [RS_OPEN_REGION-regionserver/bd6592c37bb2:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1122): Running coprocessor post-open hooks for f0252adfc27b54a76a05585f3f45bf90 2024-11-24T22:13:47,673 DEBUG [RS_OPEN_REGION-regionserver/bd6592c37bb2:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1006): Region open journal for f0252adfc27b54a76a05585f3f45bf90: Running coprocessor pre-open hook at 1732486427646Writing region info on filesystem at 1732486427646Initializing all the Stores at 1732486427647 (+1 ms)Instantiating store for column family {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732486427647Instantiating store for column family {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732486427647Instantiating store for column family {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732486427647Cleaning up temporary data from old regions at 1732486427664 (+17 ms)Running coprocessor post-open hooks at 1732486427673 (+9 ms)Region opened successfully at 1732486427673 2024-11-24T22:13:47,676 INFO [RS_OPEN_REGION-regionserver/bd6592c37bb2:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegionServer(2236): Post open deploy tasks for TestAcidGuarantees,,1732486426439.f0252adfc27b54a76a05585f3f45bf90., pid=6, masterSystemTime=1732486427638 2024-11-24T22:13:47,679 DEBUG [RS_OPEN_REGION-regionserver/bd6592c37bb2:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegionServer(2266): Finished post open deploy task for TestAcidGuarantees,,1732486426439.f0252adfc27b54a76a05585f3f45bf90. 2024-11-24T22:13:47,679 INFO [RS_OPEN_REGION-regionserver/bd6592c37bb2:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] handler.AssignRegionHandler(153): Opened TestAcidGuarantees,,1732486426439.f0252adfc27b54a76a05585f3f45bf90. 2024-11-24T22:13:47,680 INFO [PEWorker-3 {}] assignment.RegionStateStore(223): pid=5 updating hbase:meta row=f0252adfc27b54a76a05585f3f45bf90, regionState=OPEN, openSeqNum=2, regionLocation=bd6592c37bb2,40087,1732486423713 2024-11-24T22:13:47,684 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=6, ppid=5, state=RUNNABLE, hasLock=false; OpenRegionProcedure f0252adfc27b54a76a05585f3f45bf90, server=bd6592c37bb2,40087,1732486423713 because future has completed 2024-11-24T22:13:47,689 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=6, resume processing ppid=5 2024-11-24T22:13:47,689 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=6, ppid=5, state=SUCCESS, hasLock=false; OpenRegionProcedure f0252adfc27b54a76a05585f3f45bf90, server=bd6592c37bb2,40087,1732486423713 in 201 msec 2024-11-24T22:13:47,693 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=5, resume processing ppid=4 2024-11-24T22:13:47,693 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=5, ppid=4, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=f0252adfc27b54a76a05585f3f45bf90, ASSIGN in 368 msec 2024-11-24T22:13:47,694 INFO [PEWorker-1 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_UPDATE_DESC_CACHE, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_UPDATE_DESC_CACHE 2024-11-24T22:13:47,695 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1732486427694"}]},"ts":"1732486427694"} 2024-11-24T22:13:47,697 INFO [PEWorker-1 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=ENABLED in hbase:meta 2024-11-24T22:13:47,699 INFO [PEWorker-1 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_POST_OPERATION, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_POST_OPERATION 2024-11-24T22:13:47,701 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=4, state=SUCCESS, hasLock=false; CreateTableProcedure table=TestAcidGuarantees in 1.2530 sec 2024-11-24T22:13:48,612 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37701 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=4 2024-11-24T22:13:48,613 INFO [RPCClient-NioEventLoopGroup-4-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: CREATE, Table Name: default:TestAcidGuarantees completed 2024-11-24T22:13:48,620 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@2ad452c3, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-24T22:13:48,620 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request bd6592c37bb2,37701,-1 for getting cluster id 2024-11-24T22:13:48,620 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-24T22:13:48,622 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '3c461ac1-06fb-4739-8702-6368e685b03c' 2024-11-24T22:13:48,622 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-24T22:13:48,622 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "3c461ac1-06fb-4739-8702-6368e685b03c" 2024-11-24T22:13:48,623 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@ecdd502, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-24T22:13:48,623 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [bd6592c37bb2,37701,-1] 2024-11-24T22:13:48,623 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-24T22:13:48,623 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-24T22:13:48,625 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:46906, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-24T22:13:48,626 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@175b244f, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-24T22:13:48,627 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-24T22:13:48,628 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=bd6592c37bb2,40087,1732486423713, seqNum=-1] 2024-11-24T22:13:48,628 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-24T22:13:48,631 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:36512, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-24T22:13:48,633 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching master stub from registry 2024-11-24T22:13:48,634 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncConnectionImpl(321): The fetched master address is bd6592c37bb2,37701,1732486422992 2024-11-24T22:13:48,634 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.ConnectionUtils(555): The fetched master stub is org.apache.hadoop.hbase.shaded.protobuf.generated.MasterProtos$MasterService$Stub@1faa08f6 2024-11-24T22:13:48,637 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=MasterService, sasl=false 2024-11-24T22:13:48,639 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:46922, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=MasterService 2024-11-24T22:13:48,643 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@19d419ec, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-24T22:13:48,643 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request bd6592c37bb2,37701,-1 for getting cluster id 2024-11-24T22:13:48,643 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-24T22:13:48,645 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '3c461ac1-06fb-4739-8702-6368e685b03c' 2024-11-24T22:13:48,645 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-24T22:13:48,645 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "3c461ac1-06fb-4739-8702-6368e685b03c" 2024-11-24T22:13:48,646 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@43b9dd8c, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-24T22:13:48,646 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [bd6592c37bb2,37701,-1] 2024-11-24T22:13:48,646 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-24T22:13:48,647 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-24T22:13:48,648 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:46938, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-24T22:13:48,649 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@1ff1f1f0, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-24T22:13:48,649 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@130d4a2a, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-24T22:13:48,650 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request bd6592c37bb2,37701,-1 for getting cluster id 2024-11-24T22:13:48,650 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-24T22:13:48,651 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '3c461ac1-06fb-4739-8702-6368e685b03c' 2024-11-24T22:13:48,651 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-24T22:13:48,651 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "3c461ac1-06fb-4739-8702-6368e685b03c" 2024-11-24T22:13:48,652 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@4ec01134, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-24T22:13:48,652 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [bd6592c37bb2,37701,-1] 2024-11-24T22:13:48,652 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-24T22:13:48,652 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-24T22:13:48,654 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:46966, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-24T22:13:48,654 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@6b37f010, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-24T22:13:48,655 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@59d35085, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-24T22:13:48,655 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request bd6592c37bb2,37701,-1 for getting cluster id 2024-11-24T22:13:48,656 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-24T22:13:48,657 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '3c461ac1-06fb-4739-8702-6368e685b03c' 2024-11-24T22:13:48,658 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-24T22:13:48,658 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "3c461ac1-06fb-4739-8702-6368e685b03c" 2024-11-24T22:13:48,659 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@643162ed, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-24T22:13:48,659 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [bd6592c37bb2,37701,-1] 2024-11-24T22:13:48,659 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-24T22:13:48,660 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-24T22:13:48,661 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:46980, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-24T22:13:48,662 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@1380f017, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-24T22:13:48,664 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@1fb4b46b, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-24T22:13:48,664 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request bd6592c37bb2,37701,-1 for getting cluster id 2024-11-24T22:13:48,664 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-24T22:13:48,665 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '3c461ac1-06fb-4739-8702-6368e685b03c' 2024-11-24T22:13:48,666 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-24T22:13:48,666 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "3c461ac1-06fb-4739-8702-6368e685b03c" 2024-11-24T22:13:48,666 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@4c6d3ad8, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-24T22:13:48,666 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [bd6592c37bb2,37701,-1] 2024-11-24T22:13:48,666 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-24T22:13:48,667 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-24T22:13:48,668 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:47010, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-24T22:13:48,669 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@763c2183, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-24T22:13:48,670 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@3a1cce95, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-24T22:13:48,670 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request bd6592c37bb2,37701,-1 for getting cluster id 2024-11-24T22:13:48,670 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-24T22:13:48,671 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '3c461ac1-06fb-4739-8702-6368e685b03c' 2024-11-24T22:13:48,671 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-24T22:13:48,671 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "3c461ac1-06fb-4739-8702-6368e685b03c" 2024-11-24T22:13:48,671 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@2aa43e09, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-24T22:13:48,672 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [bd6592c37bb2,37701,-1] 2024-11-24T22:13:48,672 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-24T22:13:48,672 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-24T22:13:48,673 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:47024, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-24T22:13:48,674 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@28d5f584, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-24T22:13:48,676 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@3055dae7, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-24T22:13:48,676 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request bd6592c37bb2,37701,-1 for getting cluster id 2024-11-24T22:13:48,677 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-24T22:13:48,678 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '3c461ac1-06fb-4739-8702-6368e685b03c' 2024-11-24T22:13:48,678 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-24T22:13:48,678 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "3c461ac1-06fb-4739-8702-6368e685b03c" 2024-11-24T22:13:48,679 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@6c197857, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-24T22:13:48,679 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [bd6592c37bb2,37701,-1] 2024-11-24T22:13:48,679 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-24T22:13:48,679 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-24T22:13:48,680 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:47048, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-24T22:13:48,681 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@5b66dceb, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-24T22:13:48,682 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@5238acdf, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-24T22:13:48,682 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request bd6592c37bb2,37701,-1 for getting cluster id 2024-11-24T22:13:48,682 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-24T22:13:48,683 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '3c461ac1-06fb-4739-8702-6368e685b03c' 2024-11-24T22:13:48,683 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-24T22:13:48,683 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "3c461ac1-06fb-4739-8702-6368e685b03c" 2024-11-24T22:13:48,684 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@276c4225, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-24T22:13:48,684 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [bd6592c37bb2,37701,-1] 2024-11-24T22:13:48,684 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-24T22:13:48,684 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-24T22:13:48,685 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:47072, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-24T22:13:48,686 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@46f88841, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-24T22:13:48,688 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@7f4f2c18, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-24T22:13:48,688 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request bd6592c37bb2,37701,-1 for getting cluster id 2024-11-24T22:13:48,689 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-24T22:13:48,689 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '3c461ac1-06fb-4739-8702-6368e685b03c' 2024-11-24T22:13:48,690 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-24T22:13:48,690 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "3c461ac1-06fb-4739-8702-6368e685b03c" 2024-11-24T22:13:48,690 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@d0e529e, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-24T22:13:48,690 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [bd6592c37bb2,37701,-1] 2024-11-24T22:13:48,690 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-24T22:13:48,690 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-24T22:13:48,691 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:47090, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-24T22:13:48,692 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@7aaa86d6, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-24T22:13:48,693 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@6643136e, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-24T22:13:48,693 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request bd6592c37bb2,37701,-1 for getting cluster id 2024-11-24T22:13:48,693 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-24T22:13:48,694 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '3c461ac1-06fb-4739-8702-6368e685b03c' 2024-11-24T22:13:48,694 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-24T22:13:48,694 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "3c461ac1-06fb-4739-8702-6368e685b03c" 2024-11-24T22:13:48,695 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@6d25ecbc, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-24T22:13:48,695 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [bd6592c37bb2,37701,-1] 2024-11-24T22:13:48,695 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-24T22:13:48,695 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-24T22:13:48,696 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:47108, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-24T22:13:48,697 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@4a74f818, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-24T22:13:48,702 DEBUG [Thread-143 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-24T22:13:48,702 DEBUG [Thread-145 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-24T22:13:48,703 DEBUG [Thread-147 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-24T22:13:48,703 DEBUG [Thread-149 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-24T22:13:48,703 DEBUG [Thread-158 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-24T22:13:48,703 DEBUG [Thread-160 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-24T22:13:48,703 DEBUG [Thread-154 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-24T22:13:48,704 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=bd6592c37bb2,40087,1732486423713, seqNum=-1] 2024-11-24T22:13:48,705 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=bd6592c37bb2,40087,1732486423713, seqNum=-1] 2024-11-24T22:13:48,705 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=bd6592c37bb2,40087,1732486423713, seqNum=-1] 2024-11-24T22:13:48,705 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-24T22:13:48,705 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-24T22:13:48,705 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-24T22:13:48,705 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=bd6592c37bb2,40087,1732486423713, seqNum=-1] 2024-11-24T22:13:48,706 DEBUG [Thread-156 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-24T22:13:48,706 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-24T22:13:48,707 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:36518, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-24T22:13:48,707 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:36520, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-24T22:13:48,707 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:36532, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-24T22:13:48,707 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=bd6592c37bb2,40087,1732486423713, seqNum=-1] 2024-11-24T22:13:48,708 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:36548, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-24T22:13:48,708 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=bd6592c37bb2,40087,1732486423713, seqNum=-1] 2024-11-24T22:13:48,708 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-24T22:13:48,708 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-24T22:13:48,709 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=bd6592c37bb2,40087,1732486423713, seqNum=-1] 2024-11-24T22:13:48,709 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-24T22:13:48,709 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=bd6592c37bb2,40087,1732486423713, seqNum=-1] 2024-11-24T22:13:48,710 DEBUG [Thread-151 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-24T22:13:48,710 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-24T22:13:48,710 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37701 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-24T22:13:48,711 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:36570, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-24T22:13:48,711 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:36574, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-24T22:13:48,711 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:36562, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-24T22:13:48,712 DEBUG [Thread-151 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=bd6592c37bb2,40087,1732486423713, seqNum=-1] 2024-11-24T22:13:48,712 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:36584, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-24T22:13:48,712 DEBUG [Thread-151 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-24T22:13:48,713 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:36590, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-24T22:13:48,716 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37701 {}] procedure2.ProcedureExecutor(1139): Stored pid=7, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=7, table=TestAcidGuarantees 2024-11-24T22:13:48,719 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_2', locateType=CURRENT is [region=TestAcidGuarantees,,1732486426439.f0252adfc27b54a76a05585f3f45bf90., hostname=bd6592c37bb2,40087,1732486423713, seqNum=2] 2024-11-24T22:13:48,719 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='', locateType=CURRENT is [region=TestAcidGuarantees,,1732486426439.f0252adfc27b54a76a05585f3f45bf90., hostname=bd6592c37bb2,40087,1732486423713, seqNum=2] 2024-11-24T22:13:48,720 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37701 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=7 2024-11-24T22:13:48,720 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_2', locateType=CURRENT is [region=TestAcidGuarantees,,1732486426439.f0252adfc27b54a76a05585f3f45bf90., hostname=bd6592c37bb2,40087,1732486423713, seqNum=2] 2024-11-24T22:13:48,720 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_1', locateType=CURRENT is [region=TestAcidGuarantees,,1732486426439.f0252adfc27b54a76a05585f3f45bf90., hostname=bd6592c37bb2,40087,1732486423713, seqNum=2] 2024-11-24T22:13:48,721 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=7, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=7, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-24T22:13:48,724 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=7, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=7, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-24T22:13:48,725 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=8, ppid=7, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-24T22:13:48,728 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='', locateType=CURRENT is [region=TestAcidGuarantees,,1732486426439.f0252adfc27b54a76a05585f3f45bf90., hostname=bd6592c37bb2,40087,1732486423713, seqNum=2] 2024-11-24T22:13:48,733 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_1', locateType=CURRENT is [region=TestAcidGuarantees,,1732486426439.f0252adfc27b54a76a05585f3f45bf90., hostname=bd6592c37bb2,40087,1732486423713, seqNum=2] 2024-11-24T22:13:48,735 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_0', locateType=CURRENT is [region=TestAcidGuarantees,,1732486426439.f0252adfc27b54a76a05585f3f45bf90., hostname=bd6592c37bb2,40087,1732486423713, seqNum=2] 2024-11-24T22:13:48,736 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_1', locateType=CURRENT is [region=TestAcidGuarantees,,1732486426439.f0252adfc27b54a76a05585f3f45bf90., hostname=bd6592c37bb2,40087,1732486423713, seqNum=2] 2024-11-24T22:13:48,737 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_1', locateType=CURRENT is [region=TestAcidGuarantees,,1732486426439.f0252adfc27b54a76a05585f3f45bf90., hostname=bd6592c37bb2,40087,1732486423713, seqNum=2] 2024-11-24T22:13:48,792 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40087 {}] regionserver.HRegion(8855): Flush requested on f0252adfc27b54a76a05585f3f45bf90 2024-11-24T22:13:48,797 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing f0252adfc27b54a76a05585f3f45bf90 3/3 column families, dataSize=46.96 KB heapSize=140.20 KB 2024-11-24T22:13:48,815 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK f0252adfc27b54a76a05585f3f45bf90, store=A 2024-11-24T22:13:48,815 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-24T22:13:48,816 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK f0252adfc27b54a76a05585f3f45bf90, store=B 2024-11-24T22:13:48,816 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-24T22:13:48,816 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK f0252adfc27b54a76a05585f3f45bf90, store=C 2024-11-24T22:13:48,816 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-24T22:13:48,831 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37701 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=7 2024-11-24T22:13:48,912 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40087 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0252adfc27b54a76a05585f3f45bf90, server=bd6592c37bb2,40087,1732486423713 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-24T22:13:48,916 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40087 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0252adfc27b54a76a05585f3f45bf90, server=bd6592c37bb2,40087,1732486423713 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-24T22:13:48,917 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40087 {}] ipc.CallRunner(138): callId: 6 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36548 deadline: 1732486488899, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0252adfc27b54a76a05585f3f45bf90, server=bd6592c37bb2,40087,1732486423713 2024-11-24T22:13:48,919 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40087 {}] ipc.CallRunner(138): callId: 6 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36562 deadline: 1732486488900, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0252adfc27b54a76a05585f3f45bf90, server=bd6592c37bb2,40087,1732486423713 2024-11-24T22:13:48,919 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732486426439.f0252adfc27b54a76a05585f3f45bf90., hostname=bd6592c37bb2,40087,1732486423713, seqNum=2 , the old value is region=TestAcidGuarantees,,1732486426439.f0252adfc27b54a76a05585f3f45bf90., hostname=bd6592c37bb2,40087,1732486423713, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0252adfc27b54a76a05585f3f45bf90, server=bd6592c37bb2,40087,1732486423713 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T22:13:48,920 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732486426439.f0252adfc27b54a76a05585f3f45bf90., hostname=bd6592c37bb2,40087,1732486423713, seqNum=2 , the old value is region=TestAcidGuarantees,,1732486426439.f0252adfc27b54a76a05585f3f45bf90., hostname=bd6592c37bb2,40087,1732486423713, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0252adfc27b54a76a05585f3f45bf90, server=bd6592c37bb2,40087,1732486423713 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T22:13:48,920 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732486426439.f0252adfc27b54a76a05585f3f45bf90., hostname=bd6592c37bb2,40087,1732486423713, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0252adfc27b54a76a05585f3f45bf90, server=bd6592c37bb2,40087,1732486423713 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T22:13:48,920 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732486426439.f0252adfc27b54a76a05585f3f45bf90., hostname=bd6592c37bb2,40087,1732486423713, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0252adfc27b54a76a05585f3f45bf90, server=bd6592c37bb2,40087,1732486423713 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T22:13:48,920 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732486426439.f0252adfc27b54a76a05585f3f45bf90., hostname=bd6592c37bb2,40087,1732486423713, seqNum=2 because the exception is null or not the one we care about 2024-11-24T22:13:48,920 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732486426439.f0252adfc27b54a76a05585f3f45bf90., hostname=bd6592c37bb2,40087,1732486423713, seqNum=2 because the exception is null or not the one we care about 2024-11-24T22:13:48,920 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40087 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0252adfc27b54a76a05585f3f45bf90, server=bd6592c37bb2,40087,1732486423713 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-24T22:13:48,920 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40087 {}] ipc.CallRunner(138): callId: 6 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36590 deadline: 1732486488909, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0252adfc27b54a76a05585f3f45bf90, server=bd6592c37bb2,40087,1732486423713 2024-11-24T22:13:48,924 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=40087 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=8 2024-11-24T22:13:48,925 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732486426439.f0252adfc27b54a76a05585f3f45bf90., hostname=bd6592c37bb2,40087,1732486423713, seqNum=2 , the old value is region=TestAcidGuarantees,,1732486426439.f0252adfc27b54a76a05585f3f45bf90., hostname=bd6592c37bb2,40087,1732486423713, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0252adfc27b54a76a05585f3f45bf90, server=bd6592c37bb2,40087,1732486423713 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T22:13:48,925 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732486426439.f0252adfc27b54a76a05585f3f45bf90., hostname=bd6592c37bb2,40087,1732486423713, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0252adfc27b54a76a05585f3f45bf90, server=bd6592c37bb2,40087,1732486423713 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T22:13:48,925 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732486426439.f0252adfc27b54a76a05585f3f45bf90., hostname=bd6592c37bb2,40087,1732486423713, seqNum=2 because the exception is null or not the one we care about 2024-11-24T22:13:48,927 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bd6592c37bb2:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732486426439.f0252adfc27b54a76a05585f3f45bf90. 2024-11-24T22:13:48,927 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bd6592c37bb2:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732486426439.f0252adfc27b54a76a05585f3f45bf90. as already flushing 2024-11-24T22:13:48,928 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bd6592c37bb2:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732486426439.f0252adfc27b54a76a05585f3f45bf90. 2024-11-24T22:13:48,928 ERROR [RS_FLUSH_OPERATIONS-regionserver/bd6592c37bb2:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] handler.RSProcedureHandler(58): pid=8 java.io.IOException: Unable to complete flush {ENCODED => f0252adfc27b54a76a05585f3f45bf90, NAME => 'TestAcidGuarantees,,1732486426439.f0252adfc27b54a76a05585f3f45bf90.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T22:13:48,931 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40087 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0252adfc27b54a76a05585f3f45bf90, server=bd6592c37bb2,40087,1732486423713 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-24T22:13:48,932 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40087 {}] ipc.CallRunner(138): callId: 7 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36532 deadline: 1732486488917, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0252adfc27b54a76a05585f3f45bf90, server=bd6592c37bb2,40087,1732486423713 2024-11-24T22:13:48,933 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732486426439.f0252adfc27b54a76a05585f3f45bf90., hostname=bd6592c37bb2,40087,1732486423713, seqNum=2 , the old value is region=TestAcidGuarantees,,1732486426439.f0252adfc27b54a76a05585f3f45bf90., hostname=bd6592c37bb2,40087,1732486423713, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0252adfc27b54a76a05585f3f45bf90, server=bd6592c37bb2,40087,1732486423713 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T22:13:48,933 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732486426439.f0252adfc27b54a76a05585f3f45bf90., hostname=bd6592c37bb2,40087,1732486423713, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0252adfc27b54a76a05585f3f45bf90, server=bd6592c37bb2,40087,1732486423713 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T22:13:48,933 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732486426439.f0252adfc27b54a76a05585f3f45bf90., hostname=bd6592c37bb2,40087,1732486423713, seqNum=2 because the exception is null or not the one we care about 2024-11-24T22:13:48,934 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40087 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0252adfc27b54a76a05585f3f45bf90, server=bd6592c37bb2,40087,1732486423713 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-24T22:13:48,935 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40087 {}] ipc.CallRunner(138): callId: 7 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36518 deadline: 1732486488917, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0252adfc27b54a76a05585f3f45bf90, server=bd6592c37bb2,40087,1732486423713 2024-11-24T22:13:48,935 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732486426439.f0252adfc27b54a76a05585f3f45bf90., hostname=bd6592c37bb2,40087,1732486423713, seqNum=2 , the old value is region=TestAcidGuarantees,,1732486426439.f0252adfc27b54a76a05585f3f45bf90., hostname=bd6592c37bb2,40087,1732486423713, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0252adfc27b54a76a05585f3f45bf90, server=bd6592c37bb2,40087,1732486423713 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T22:13:48,935 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/f0252adfc27b54a76a05585f3f45bf90/.tmp/A/85bc3afcd03143c1b1f0def65f0e2e63 is 50, key is test_row_0/A:col10/1732486428794/Put/seqid=0 2024-11-24T22:13:48,935 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732486426439.f0252adfc27b54a76a05585f3f45bf90., hostname=bd6592c37bb2,40087,1732486423713, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0252adfc27b54a76a05585f3f45bf90, server=bd6592c37bb2,40087,1732486423713 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T22:13:48,935 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732486426439.f0252adfc27b54a76a05585f3f45bf90., hostname=bd6592c37bb2,40087,1732486423713, seqNum=2 because the exception is null or not the one we care about 2024-11-24T22:13:48,943 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bd6592c37bb2:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=8 java.io.IOException: Unable to complete flush {ENCODED => f0252adfc27b54a76a05585f3f45bf90, NAME => 'TestAcidGuarantees,,1732486426439.f0252adfc27b54a76a05585f3f45bf90.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T22:13:48,954 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45929 is added to blk_1073741837_1013 (size=14341) 2024-11-24T22:13:48,956 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=14 (bloomFilter=true), to=hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/f0252adfc27b54a76a05585f3f45bf90/.tmp/A/85bc3afcd03143c1b1f0def65f0e2e63 2024-11-24T22:13:48,961 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37701 {}] master.HMaster(4177): Remote procedure failed, pid=8 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => f0252adfc27b54a76a05585f3f45bf90, NAME => 'TestAcidGuarantees,,1732486426439.f0252adfc27b54a76a05585f3f45bf90.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => f0252adfc27b54a76a05585f3f45bf90, NAME => 'TestAcidGuarantees,,1732486426439.f0252adfc27b54a76a05585f3f45bf90.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T22:13:49,038 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40087 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0252adfc27b54a76a05585f3f45bf90, server=bd6592c37bb2,40087,1732486423713 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-24T22:13:49,039 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40087 {}] ipc.CallRunner(138): callId: 7 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36590 deadline: 1732486489033, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0252adfc27b54a76a05585f3f45bf90, server=bd6592c37bb2,40087,1732486423713 2024-11-24T22:13:49,040 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732486426439.f0252adfc27b54a76a05585f3f45bf90., hostname=bd6592c37bb2,40087,1732486423713, seqNum=2 , the old value is region=TestAcidGuarantees,,1732486426439.f0252adfc27b54a76a05585f3f45bf90., hostname=bd6592c37bb2,40087,1732486423713, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0252adfc27b54a76a05585f3f45bf90, server=bd6592c37bb2,40087,1732486423713 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T22:13:49,040 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732486426439.f0252adfc27b54a76a05585f3f45bf90., hostname=bd6592c37bb2,40087,1732486423713, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0252adfc27b54a76a05585f3f45bf90, server=bd6592c37bb2,40087,1732486423713 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T22:13:49,040 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732486426439.f0252adfc27b54a76a05585f3f45bf90., hostname=bd6592c37bb2,40087,1732486423713, seqNum=2 because the exception is null or not the one we care about 2024-11-24T22:13:49,040 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40087 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0252adfc27b54a76a05585f3f45bf90, server=bd6592c37bb2,40087,1732486423713 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-24T22:13:49,040 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40087 {}] ipc.CallRunner(138): callId: 7 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36548 deadline: 1732486489033, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0252adfc27b54a76a05585f3f45bf90, server=bd6592c37bb2,40087,1732486423713 2024-11-24T22:13:49,041 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40087 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0252adfc27b54a76a05585f3f45bf90, server=bd6592c37bb2,40087,1732486423713 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-24T22:13:49,042 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40087 {}] ipc.CallRunner(138): callId: 7 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36562 deadline: 1732486489033, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0252adfc27b54a76a05585f3f45bf90, server=bd6592c37bb2,40087,1732486423713 2024-11-24T22:13:49,043 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37701 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=7 2024-11-24T22:13:49,045 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732486426439.f0252adfc27b54a76a05585f3f45bf90., hostname=bd6592c37bb2,40087,1732486423713, seqNum=2 , the old value is region=TestAcidGuarantees,,1732486426439.f0252adfc27b54a76a05585f3f45bf90., hostname=bd6592c37bb2,40087,1732486423713, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0252adfc27b54a76a05585f3f45bf90, server=bd6592c37bb2,40087,1732486423713 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T22:13:49,045 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732486426439.f0252adfc27b54a76a05585f3f45bf90., hostname=bd6592c37bb2,40087,1732486423713, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0252adfc27b54a76a05585f3f45bf90, server=bd6592c37bb2,40087,1732486423713 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T22:13:49,045 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732486426439.f0252adfc27b54a76a05585f3f45bf90., hostname=bd6592c37bb2,40087,1732486423713, seqNum=2 because the exception is null or not the one we care about 2024-11-24T22:13:49,046 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732486426439.f0252adfc27b54a76a05585f3f45bf90., hostname=bd6592c37bb2,40087,1732486423713, seqNum=2 , the old value is region=TestAcidGuarantees,,1732486426439.f0252adfc27b54a76a05585f3f45bf90., hostname=bd6592c37bb2,40087,1732486423713, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0252adfc27b54a76a05585f3f45bf90, server=bd6592c37bb2,40087,1732486423713 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T22:13:49,046 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732486426439.f0252adfc27b54a76a05585f3f45bf90., hostname=bd6592c37bb2,40087,1732486423713, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0252adfc27b54a76a05585f3f45bf90, server=bd6592c37bb2,40087,1732486423713 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T22:13:49,046 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732486426439.f0252adfc27b54a76a05585f3f45bf90., hostname=bd6592c37bb2,40087,1732486423713, seqNum=2 because the exception is null or not the one we care about 2024-11-24T22:13:49,048 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40087 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0252adfc27b54a76a05585f3f45bf90, server=bd6592c37bb2,40087,1732486423713 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-24T22:13:49,049 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40087 {}] ipc.CallRunner(138): callId: 8 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36532 deadline: 1732486489042, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0252adfc27b54a76a05585f3f45bf90, server=bd6592c37bb2,40087,1732486423713 2024-11-24T22:13:49,049 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732486426439.f0252adfc27b54a76a05585f3f45bf90., hostname=bd6592c37bb2,40087,1732486423713, seqNum=2 , the old value is region=TestAcidGuarantees,,1732486426439.f0252adfc27b54a76a05585f3f45bf90., hostname=bd6592c37bb2,40087,1732486423713, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0252adfc27b54a76a05585f3f45bf90, server=bd6592c37bb2,40087,1732486423713 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T22:13:49,050 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732486426439.f0252adfc27b54a76a05585f3f45bf90., hostname=bd6592c37bb2,40087,1732486423713, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0252adfc27b54a76a05585f3f45bf90, server=bd6592c37bb2,40087,1732486423713 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T22:13:49,050 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732486426439.f0252adfc27b54a76a05585f3f45bf90., hostname=bd6592c37bb2,40087,1732486423713, seqNum=2 because the exception is null or not the one we care about 2024-11-24T22:13:49,050 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40087 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0252adfc27b54a76a05585f3f45bf90, server=bd6592c37bb2,40087,1732486423713 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-24T22:13:49,050 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40087 {}] ipc.CallRunner(138): callId: 8 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36518 deadline: 1732486489044, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0252adfc27b54a76a05585f3f45bf90, server=bd6592c37bb2,40087,1732486423713 2024-11-24T22:13:49,051 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732486426439.f0252adfc27b54a76a05585f3f45bf90., hostname=bd6592c37bb2,40087,1732486423713, seqNum=2 , the old value is region=TestAcidGuarantees,,1732486426439.f0252adfc27b54a76a05585f3f45bf90., hostname=bd6592c37bb2,40087,1732486423713, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0252adfc27b54a76a05585f3f45bf90, server=bd6592c37bb2,40087,1732486423713 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T22:13:49,051 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732486426439.f0252adfc27b54a76a05585f3f45bf90., hostname=bd6592c37bb2,40087,1732486423713, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0252adfc27b54a76a05585f3f45bf90, server=bd6592c37bb2,40087,1732486423713 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T22:13:49,051 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732486426439.f0252adfc27b54a76a05585f3f45bf90., hostname=bd6592c37bb2,40087,1732486423713, seqNum=2 because the exception is null or not the one we care about 2024-11-24T22:13:49,060 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/f0252adfc27b54a76a05585f3f45bf90/.tmp/B/fa858762490548298016038bb1fe3609 is 50, key is test_row_0/B:col10/1732486428794/Put/seqid=0 2024-11-24T22:13:49,083 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45929 is added to blk_1073741838_1014 (size=12001) 2024-11-24T22:13:49,086 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=14 (bloomFilter=true), to=hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/f0252adfc27b54a76a05585f3f45bf90/.tmp/B/fa858762490548298016038bb1fe3609 2024-11-24T22:13:49,115 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=40087 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=8 2024-11-24T22:13:49,128 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bd6592c37bb2:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732486426439.f0252adfc27b54a76a05585f3f45bf90. 2024-11-24T22:13:49,128 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bd6592c37bb2:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732486426439.f0252adfc27b54a76a05585f3f45bf90. as already flushing 2024-11-24T22:13:49,129 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bd6592c37bb2:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732486426439.f0252adfc27b54a76a05585f3f45bf90. 2024-11-24T22:13:49,129 ERROR [RS_FLUSH_OPERATIONS-regionserver/bd6592c37bb2:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] handler.RSProcedureHandler(58): pid=8 java.io.IOException: Unable to complete flush {ENCODED => f0252adfc27b54a76a05585f3f45bf90, NAME => 'TestAcidGuarantees,,1732486426439.f0252adfc27b54a76a05585f3f45bf90.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T22:13:49,129 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bd6592c37bb2:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=8 java.io.IOException: Unable to complete flush {ENCODED => f0252adfc27b54a76a05585f3f45bf90, NAME => 'TestAcidGuarantees,,1732486426439.f0252adfc27b54a76a05585f3f45bf90.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T22:13:49,130 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37701 {}] master.HMaster(4177): Remote procedure failed, pid=8 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => f0252adfc27b54a76a05585f3f45bf90, NAME => 'TestAcidGuarantees,,1732486426439.f0252adfc27b54a76a05585f3f45bf90.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => f0252adfc27b54a76a05585f3f45bf90, NAME => 'TestAcidGuarantees,,1732486426439.f0252adfc27b54a76a05585f3f45bf90.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T22:13:49,157 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/f0252adfc27b54a76a05585f3f45bf90/.tmp/C/6cb278bb60bd4562b947a68e3f06567b is 50, key is test_row_0/C:col10/1732486428794/Put/seqid=0 2024-11-24T22:13:49,183 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45929 is added to blk_1073741839_1015 (size=12001) 2024-11-24T22:13:49,185 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=14 (bloomFilter=true), to=hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/f0252adfc27b54a76a05585f3f45bf90/.tmp/C/6cb278bb60bd4562b947a68e3f06567b 2024-11-24T22:13:49,199 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/f0252adfc27b54a76a05585f3f45bf90/.tmp/A/85bc3afcd03143c1b1f0def65f0e2e63 as hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/f0252adfc27b54a76a05585f3f45bf90/A/85bc3afcd03143c1b1f0def65f0e2e63 2024-11-24T22:13:49,211 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/f0252adfc27b54a76a05585f3f45bf90/A/85bc3afcd03143c1b1f0def65f0e2e63, entries=200, sequenceid=14, filesize=14.0 K 2024-11-24T22:13:49,220 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/f0252adfc27b54a76a05585f3f45bf90/.tmp/B/fa858762490548298016038bb1fe3609 as hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/f0252adfc27b54a76a05585f3f45bf90/B/fa858762490548298016038bb1fe3609 2024-11-24T22:13:49,235 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/f0252adfc27b54a76a05585f3f45bf90/B/fa858762490548298016038bb1fe3609, entries=150, sequenceid=14, filesize=11.7 K 2024-11-24T22:13:49,241 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/f0252adfc27b54a76a05585f3f45bf90/.tmp/C/6cb278bb60bd4562b947a68e3f06567b as hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/f0252adfc27b54a76a05585f3f45bf90/C/6cb278bb60bd4562b947a68e3f06567b 2024-11-24T22:13:49,258 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40087 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0252adfc27b54a76a05585f3f45bf90, server=bd6592c37bb2,40087,1732486423713 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-24T22:13:49,259 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40087 {}] ipc.CallRunner(138): callId: 8 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36590 deadline: 1732486489252, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0252adfc27b54a76a05585f3f45bf90, server=bd6592c37bb2,40087,1732486423713 2024-11-24T22:13:49,260 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732486426439.f0252adfc27b54a76a05585f3f45bf90., hostname=bd6592c37bb2,40087,1732486423713, seqNum=2 , the old value is region=TestAcidGuarantees,,1732486426439.f0252adfc27b54a76a05585f3f45bf90., hostname=bd6592c37bb2,40087,1732486423713, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0252adfc27b54a76a05585f3f45bf90, server=bd6592c37bb2,40087,1732486423713 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T22:13:49,260 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732486426439.f0252adfc27b54a76a05585f3f45bf90., hostname=bd6592c37bb2,40087,1732486423713, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0252adfc27b54a76a05585f3f45bf90, server=bd6592c37bb2,40087,1732486423713 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T22:13:49,260 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732486426439.f0252adfc27b54a76a05585f3f45bf90., hostname=bd6592c37bb2,40087,1732486423713, seqNum=2 because the exception is null or not the one we care about 2024-11-24T22:13:49,260 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/f0252adfc27b54a76a05585f3f45bf90/C/6cb278bb60bd4562b947a68e3f06567b, entries=150, sequenceid=14, filesize=11.7 K 2024-11-24T22:13:49,260 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40087 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0252adfc27b54a76a05585f3f45bf90, server=bd6592c37bb2,40087,1732486423713 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-24T22:13:49,260 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40087 {}] ipc.CallRunner(138): callId: 8 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36548 deadline: 1732486489253, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0252adfc27b54a76a05585f3f45bf90, server=bd6592c37bb2,40087,1732486423713 2024-11-24T22:13:49,261 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732486426439.f0252adfc27b54a76a05585f3f45bf90., hostname=bd6592c37bb2,40087,1732486423713, seqNum=2 , the old value is region=TestAcidGuarantees,,1732486426439.f0252adfc27b54a76a05585f3f45bf90., hostname=bd6592c37bb2,40087,1732486423713, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0252adfc27b54a76a05585f3f45bf90, server=bd6592c37bb2,40087,1732486423713 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T22:13:49,261 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732486426439.f0252adfc27b54a76a05585f3f45bf90., hostname=bd6592c37bb2,40087,1732486423713, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0252adfc27b54a76a05585f3f45bf90, server=bd6592c37bb2,40087,1732486423713 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T22:13:49,261 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732486426439.f0252adfc27b54a76a05585f3f45bf90., hostname=bd6592c37bb2,40087,1732486423713, seqNum=2 because the exception is null or not the one we care about 2024-11-24T22:13:49,268 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~67.09 KB/68700, heapSize ~199.92 KB/204720, currentSize=114.05 KB/116790 for f0252adfc27b54a76a05585f3f45bf90 in 470ms, sequenceid=14, compaction requested=false 2024-11-24T22:13:49,270 DEBUG [MemStoreFlusher.0 {}] regionserver.MetricsTableSourceImpl(133): Creating new MetricsTableSourceImpl for table 'TestAcidGuarantees' 2024-11-24T22:13:49,272 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for f0252adfc27b54a76a05585f3f45bf90: 2024-11-24T22:13:49,277 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/bd6592c37bb2:0-0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1732486428890, max timestamp=1732486429262Num uniques 859; 2024-11-24T22:13:49,281 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/bd6592c37bb2:0-2 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1732486428890, max timestamp=1732486429262Num uniques 859; 2024-11-24T22:13:49,282 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40087 {}] regionserver.HRegion(8855): Flush requested on f0252adfc27b54a76a05585f3f45bf90 2024-11-24T22:13:49,284 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing f0252adfc27b54a76a05585f3f45bf90 3/3 column families, dataSize=64.85 KB heapSize=184.09 KB 2024-11-24T22:13:49,281 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/bd6592c37bb2:0-0 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-11-24T22:13:49,281 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/bd6592c37bb2:0-2 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-11-24T22:13:49,285 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/bd6592c37bb2:0-1 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1732486428890, max timestamp=1732486429262Num uniques 859; 2024-11-24T22:13:49,285 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/bd6592c37bb2:0-1 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-11-24T22:13:49,286 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=40087 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=8 2024-11-24T22:13:49,287 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK f0252adfc27b54a76a05585f3f45bf90, store=A 2024-11-24T22:13:49,288 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bd6592c37bb2:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732486426439.f0252adfc27b54a76a05585f3f45bf90. 2024-11-24T22:13:49,288 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bd6592c37bb2:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732486426439.f0252adfc27b54a76a05585f3f45bf90. as already flushing 2024-11-24T22:13:49,288 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bd6592c37bb2:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732486426439.f0252adfc27b54a76a05585f3f45bf90. 2024-11-24T22:13:49,289 ERROR [RS_FLUSH_OPERATIONS-regionserver/bd6592c37bb2:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] handler.RSProcedureHandler(58): pid=8 java.io.IOException: Unable to complete flush {ENCODED => f0252adfc27b54a76a05585f3f45bf90, NAME => 'TestAcidGuarantees,,1732486426439.f0252adfc27b54a76a05585f3f45bf90.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T22:13:49,289 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bd6592c37bb2:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=8 java.io.IOException: Unable to complete flush {ENCODED => f0252adfc27b54a76a05585f3f45bf90, NAME => 'TestAcidGuarantees,,1732486426439.f0252adfc27b54a76a05585f3f45bf90.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T22:13:49,290 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-24T22:13:49,290 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK f0252adfc27b54a76a05585f3f45bf90, store=B 2024-11-24T22:13:49,290 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-24T22:13:49,290 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK f0252adfc27b54a76a05585f3f45bf90, store=C 2024-11-24T22:13:49,290 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-24T22:13:49,291 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37701 {}] master.HMaster(4177): Remote procedure failed, pid=8 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => f0252adfc27b54a76a05585f3f45bf90, NAME => 'TestAcidGuarantees,,1732486426439.f0252adfc27b54a76a05585f3f45bf90.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => f0252adfc27b54a76a05585f3f45bf90, NAME => 'TestAcidGuarantees,,1732486426439.f0252adfc27b54a76a05585f3f45bf90.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T22:13:49,313 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/f0252adfc27b54a76a05585f3f45bf90/.tmp/A/c148305567ff449293163aa4d9ca6086 is 50, key is test_row_0/A:col10/1732486429279/Put/seqid=0 2024-11-24T22:13:49,335 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45929 is added to blk_1073741840_1016 (size=19021) 2024-11-24T22:13:49,336 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=13.42 KB at sequenceid=37 (bloomFilter=true), to=hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/f0252adfc27b54a76a05585f3f45bf90/.tmp/A/c148305567ff449293163aa4d9ca6086 2024-11-24T22:13:49,351 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37701 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=7 2024-11-24T22:13:49,359 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/f0252adfc27b54a76a05585f3f45bf90/.tmp/B/b7e5d8cf64e84135afbc9fb3af466717 is 50, key is test_row_0/B:col10/1732486429279/Put/seqid=0 2024-11-24T22:13:49,385 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45929 is added to blk_1073741841_1017 (size=12001) 2024-11-24T22:13:49,386 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=13.42 KB at sequenceid=37 (bloomFilter=true), to=hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/f0252adfc27b54a76a05585f3f45bf90/.tmp/B/b7e5d8cf64e84135afbc9fb3af466717 2024-11-24T22:13:49,409 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/f0252adfc27b54a76a05585f3f45bf90/.tmp/C/cb90eafdaeba493ea99a4ef7d255056f is 50, key is test_row_0/C:col10/1732486429279/Put/seqid=0 2024-11-24T22:13:49,429 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45929 is added to blk_1073741842_1018 (size=12001) 2024-11-24T22:13:49,438 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/bd6592c37bb2:0-3 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=209, cellSize=9564, totalHeapSize=21976, min timestamp=1732486429401, max timestamp=1732486429424Num uniques 859; 2024-11-24T22:13:49,438 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/bd6592c37bb2:0-3 {}] regionserver.CompactionPipeline(163): Suffix data size=39334, new segment data size=9564, suffix heap size=117064,new segment heap size=21976 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=209 2024-11-24T22:13:49,442 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/bd6592c37bb2:0-4 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=209, cellSize=9564, totalHeapSize=21976, min timestamp=1732486429401, max timestamp=1732486429424Num uniques 859; 2024-11-24T22:13:49,444 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=40087 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=8 2024-11-24T22:13:49,444 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bd6592c37bb2:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732486426439.f0252adfc27b54a76a05585f3f45bf90. 2024-11-24T22:13:49,445 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bd6592c37bb2:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732486426439.f0252adfc27b54a76a05585f3f45bf90. as already flushing 2024-11-24T22:13:49,445 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bd6592c37bb2:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732486426439.f0252adfc27b54a76a05585f3f45bf90. 2024-11-24T22:13:49,445 ERROR [RS_FLUSH_OPERATIONS-regionserver/bd6592c37bb2:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] handler.RSProcedureHandler(58): pid=8 java.io.IOException: Unable to complete flush {ENCODED => f0252adfc27b54a76a05585f3f45bf90, NAME => 'TestAcidGuarantees,,1732486426439.f0252adfc27b54a76a05585f3f45bf90.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T22:13:49,445 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bd6592c37bb2:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=8 java.io.IOException: Unable to complete flush {ENCODED => f0252adfc27b54a76a05585f3f45bf90, NAME => 'TestAcidGuarantees,,1732486426439.f0252adfc27b54a76a05585f3f45bf90.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T22:13:49,447 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/bd6592c37bb2:0-5 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=209, cellSize=9565, totalHeapSize=21976, min timestamp=1732486429401, max timestamp=1732486429424Num uniques 859; 2024-11-24T22:13:49,447 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37701 {}] master.HMaster(4177): Remote procedure failed, pid=8 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => f0252adfc27b54a76a05585f3f45bf90, NAME => 'TestAcidGuarantees,,1732486426439.f0252adfc27b54a76a05585f3f45bf90.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => f0252adfc27b54a76a05585f3f45bf90, NAME => 'TestAcidGuarantees,,1732486426439.f0252adfc27b54a76a05585f3f45bf90.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T22:13:49,444 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/bd6592c37bb2:0-4 {}] regionserver.CompactionPipeline(163): Suffix data size=39334, new segment data size=9564, suffix heap size=117064,new segment heap size=21976 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=209 2024-11-24T22:13:49,447 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/bd6592c37bb2:0-5 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=9565, suffix heap size=117064,new segment heap size=21976 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=209 2024-11-24T22:13:49,602 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=40087 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=8 2024-11-24T22:13:49,602 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bd6592c37bb2:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732486426439.f0252adfc27b54a76a05585f3f45bf90. 2024-11-24T22:13:49,602 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bd6592c37bb2:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732486426439.f0252adfc27b54a76a05585f3f45bf90. as already flushing 2024-11-24T22:13:49,602 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bd6592c37bb2:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732486426439.f0252adfc27b54a76a05585f3f45bf90. 2024-11-24T22:13:49,602 ERROR [RS_FLUSH_OPERATIONS-regionserver/bd6592c37bb2:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] handler.RSProcedureHandler(58): pid=8 java.io.IOException: Unable to complete flush {ENCODED => f0252adfc27b54a76a05585f3f45bf90, NAME => 'TestAcidGuarantees,,1732486426439.f0252adfc27b54a76a05585f3f45bf90.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T22:13:49,603 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bd6592c37bb2:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=8 java.io.IOException: Unable to complete flush {ENCODED => f0252adfc27b54a76a05585f3f45bf90, NAME => 'TestAcidGuarantees,,1732486426439.f0252adfc27b54a76a05585f3f45bf90.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T22:13:49,605 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37701 {}] master.HMaster(4177): Remote procedure failed, pid=8 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => f0252adfc27b54a76a05585f3f45bf90, NAME => 'TestAcidGuarantees,,1732486426439.f0252adfc27b54a76a05585f3f45bf90.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => f0252adfc27b54a76a05585f3f45bf90, NAME => 'TestAcidGuarantees,,1732486426439.f0252adfc27b54a76a05585f3f45bf90.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T22:13:49,608 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40087 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0252adfc27b54a76a05585f3f45bf90, server=bd6592c37bb2,40087,1732486423713 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-24T22:13:49,609 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40087 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36548 deadline: 1732486489573, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0252adfc27b54a76a05585f3f45bf90, server=bd6592c37bb2,40087,1732486423713 2024-11-24T22:13:49,609 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/bd6592c37bb2:0-7 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=218, cellSize=9978, totalHeapSize=22912, min timestamp=1732486429563, max timestamp=1732486429597Num uniques 1068; 2024-11-24T22:13:49,610 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/bd6592c37bb2:0-7 {}] regionserver.CompactionPipeline(163): Suffix data size=48908, new segment data size=9978, suffix heap size=139040,new segment heap size=22912 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1068, new segment cells count=218 2024-11-24T22:13:49,610 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732486426439.f0252adfc27b54a76a05585f3f45bf90., hostname=bd6592c37bb2,40087,1732486423713, seqNum=2 , the old value is region=TestAcidGuarantees,,1732486426439.f0252adfc27b54a76a05585f3f45bf90., hostname=bd6592c37bb2,40087,1732486423713, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0252adfc27b54a76a05585f3f45bf90, server=bd6592c37bb2,40087,1732486423713 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T22:13:49,610 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732486426439.f0252adfc27b54a76a05585f3f45bf90., hostname=bd6592c37bb2,40087,1732486423713, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0252adfc27b54a76a05585f3f45bf90, server=bd6592c37bb2,40087,1732486423713 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T22:13:49,611 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732486426439.f0252adfc27b54a76a05585f3f45bf90., hostname=bd6592c37bb2,40087,1732486423713, seqNum=2 because the exception is null or not the one we care about 2024-11-24T22:13:49,612 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/bd6592c37bb2:0-6 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=368, cellSize=16848, totalHeapSize=38512, min timestamp=1732486429506, max timestamp=1732486429597Num uniques 1068; 2024-11-24T22:13:49,613 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/bd6592c37bb2:0-6 {}] regionserver.CompactionPipeline(163): Suffix data size=48908, new segment data size=16848, suffix heap size=139040,new segment heap size=38512 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1068, new segment cells count=368 2024-11-24T22:13:49,628 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/bd6592c37bb2:0-8 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=150, cellSize=6870, totalHeapSize=15840, min timestamp=1732486429567, max timestamp=1732486429597Num uniques 1068; 2024-11-24T22:13:49,628 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/bd6592c37bb2:0-8 {}] regionserver.CompactionPipeline(163): Suffix data size=48908, new segment data size=6870, suffix heap size=139040,new segment heap size=15840 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1068, new segment cells count=150 2024-11-24T22:13:49,722 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/bd6592c37bb2:0-2 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=277, cellSize=12682, totalHeapSize=29048, min timestamp=1732486429671, max timestamp=1732486429712Num uniques 1009; 2024-11-24T22:13:49,722 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/bd6592c37bb2:0-2 {}] regionserver.CompactionPipeline(163): Suffix data size=46214, new segment data size=12682, suffix heap size=132904,new segment heap size=29048 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1009, new segment cells count=277 2024-11-24T22:13:49,725 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/bd6592c37bb2:0-0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=277, cellSize=12682, totalHeapSize=29048, min timestamp=1732486429671, max timestamp=1732486429712Num uniques 1077; 2024-11-24T22:13:49,726 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/bd6592c37bb2:0-0 {}] regionserver.CompactionPipeline(163): Suffix data size=49322, new segment data size=12682, suffix heap size=139976,new segment heap size=29048 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1077, new segment cells count=277 2024-11-24T22:13:49,733 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/bd6592c37bb2:0-9 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=277, cellSize=12682, totalHeapSize=29048, min timestamp=1732486429671, max timestamp=1732486429712Num uniques 1227; 2024-11-24T22:13:49,733 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/bd6592c37bb2:0-9 {}] regionserver.CompactionPipeline(163): Suffix data size=56192, new segment data size=12682, suffix heap size=155576,new segment heap size=29048 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1227, new segment cells count=277 2024-11-24T22:13:49,759 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=40087 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=8 2024-11-24T22:13:49,759 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bd6592c37bb2:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732486426439.f0252adfc27b54a76a05585f3f45bf90. 2024-11-24T22:13:49,759 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bd6592c37bb2:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732486426439.f0252adfc27b54a76a05585f3f45bf90. as already flushing 2024-11-24T22:13:49,759 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bd6592c37bb2:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732486426439.f0252adfc27b54a76a05585f3f45bf90. 2024-11-24T22:13:49,759 ERROR [RS_FLUSH_OPERATIONS-regionserver/bd6592c37bb2:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] handler.RSProcedureHandler(58): pid=8 java.io.IOException: Unable to complete flush {ENCODED => f0252adfc27b54a76a05585f3f45bf90, NAME => 'TestAcidGuarantees,,1732486426439.f0252adfc27b54a76a05585f3f45bf90.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T22:13:49,760 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bd6592c37bb2:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=8 java.io.IOException: Unable to complete flush {ENCODED => f0252adfc27b54a76a05585f3f45bf90, NAME => 'TestAcidGuarantees,,1732486426439.f0252adfc27b54a76a05585f3f45bf90.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T22:13:49,761 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37701 {}] master.HMaster(4177): Remote procedure failed, pid=8 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => f0252adfc27b54a76a05585f3f45bf90, NAME => 'TestAcidGuarantees,,1732486426439.f0252adfc27b54a76a05585f3f45bf90.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => f0252adfc27b54a76a05585f3f45bf90, NAME => 'TestAcidGuarantees,,1732486426439.f0252adfc27b54a76a05585f3f45bf90.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T22:13:49,831 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=13.42 KB at sequenceid=37 (bloomFilter=true), to=hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/f0252adfc27b54a76a05585f3f45bf90/.tmp/C/cb90eafdaeba493ea99a4ef7d255056f 2024-11-24T22:13:49,835 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40087 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0252adfc27b54a76a05585f3f45bf90, server=bd6592c37bb2,40087,1732486423713 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-24T22:13:49,835 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40087 {}] ipc.CallRunner(138): callId: 18 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36590 deadline: 1732486489804, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0252adfc27b54a76a05585f3f45bf90, server=bd6592c37bb2,40087,1732486423713 2024-11-24T22:13:49,836 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732486426439.f0252adfc27b54a76a05585f3f45bf90., hostname=bd6592c37bb2,40087,1732486423713, seqNum=2 , the old value is region=TestAcidGuarantees,,1732486426439.f0252adfc27b54a76a05585f3f45bf90., hostname=bd6592c37bb2,40087,1732486423713, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0252adfc27b54a76a05585f3f45bf90, server=bd6592c37bb2,40087,1732486423713 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T22:13:49,837 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732486426439.f0252adfc27b54a76a05585f3f45bf90., hostname=bd6592c37bb2,40087,1732486423713, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0252adfc27b54a76a05585f3f45bf90, server=bd6592c37bb2,40087,1732486423713 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T22:13:49,837 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732486426439.f0252adfc27b54a76a05585f3f45bf90., hostname=bd6592c37bb2,40087,1732486423713, seqNum=2 because the exception is null or not the one we care about 2024-11-24T22:13:49,837 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/bd6592c37bb2:0-3 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=236, cellSize=10806, totalHeapSize=24784, min timestamp=1732486429779, max timestamp=1732486429804Num uniques 1136; 2024-11-24T22:13:49,837 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/bd6592c37bb2:0-3 {}] regionserver.CompactionPipeline(163): Suffix data size=52026, new segment data size=10806, suffix heap size=146112,new segment heap size=24784 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1136, new segment cells count=236 2024-11-24T22:13:49,837 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/bd6592c37bb2:0-1 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=236, cellSize=10806, totalHeapSize=24784, min timestamp=1732486429779, max timestamp=1732486429804Num uniques 1136; 2024-11-24T22:13:49,838 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/bd6592c37bb2:0-4 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=236, cellSize=10806, totalHeapSize=24784, min timestamp=1732486429779, max timestamp=1732486429804Num uniques 1136; 2024-11-24T22:13:49,838 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/bd6592c37bb2:0-1 {}] regionserver.CompactionPipeline(163): Suffix data size=52026, new segment data size=10806, suffix heap size=146112,new segment heap size=24784 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1136, new segment cells count=236 2024-11-24T22:13:49,838 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/bd6592c37bb2:0-4 {}] regionserver.CompactionPipeline(163): Suffix data size=52026, new segment data size=10806, suffix heap size=146112,new segment heap size=24784 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1136, new segment cells count=236 2024-11-24T22:13:49,844 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/f0252adfc27b54a76a05585f3f45bf90/.tmp/A/c148305567ff449293163aa4d9ca6086 as hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/f0252adfc27b54a76a05585f3f45bf90/A/c148305567ff449293163aa4d9ca6086 2024-11-24T22:13:49,861 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37701 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=7 2024-11-24T22:13:49,861 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/f0252adfc27b54a76a05585f3f45bf90/A/c148305567ff449293163aa4d9ca6086, entries=300, sequenceid=37, filesize=18.6 K 2024-11-24T22:13:49,864 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/f0252adfc27b54a76a05585f3f45bf90/.tmp/B/b7e5d8cf64e84135afbc9fb3af466717 as hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/f0252adfc27b54a76a05585f3f45bf90/B/b7e5d8cf64e84135afbc9fb3af466717 2024-11-24T22:13:49,888 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/f0252adfc27b54a76a05585f3f45bf90/B/b7e5d8cf64e84135afbc9fb3af466717, entries=150, sequenceid=37, filesize=11.7 K 2024-11-24T22:13:49,890 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/f0252adfc27b54a76a05585f3f45bf90/.tmp/C/cb90eafdaeba493ea99a4ef7d255056f as hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/f0252adfc27b54a76a05585f3f45bf90/C/cb90eafdaeba493ea99a4ef7d255056f 2024-11-24T22:13:49,906 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/f0252adfc27b54a76a05585f3f45bf90/C/cb90eafdaeba493ea99a4ef7d255056f, entries=150, sequenceid=37, filesize=11.7 K 2024-11-24T22:13:49,908 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~40.25 KB/41220, heapSize ~106.03 KB/108576, currentSize=93.93 KB/96180 for f0252adfc27b54a76a05585f3f45bf90 in 624ms, sequenceid=37, compaction requested=false 2024-11-24T22:13:49,908 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for f0252adfc27b54a76a05585f3f45bf90: 2024-11-24T22:13:49,914 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40087 {}] regionserver.HRegion(8855): Flush requested on f0252adfc27b54a76a05585f3f45bf90 2024-11-24T22:13:49,915 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing f0252adfc27b54a76a05585f3f45bf90 3/3 column families, dataSize=100.63 KB heapSize=278.16 KB 2024-11-24T22:13:49,915 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=40087 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=8 2024-11-24T22:13:49,915 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bd6592c37bb2:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732486426439.f0252adfc27b54a76a05585f3f45bf90. 2024-11-24T22:13:49,915 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bd6592c37bb2:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732486426439.f0252adfc27b54a76a05585f3f45bf90. as already flushing 2024-11-24T22:13:49,916 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bd6592c37bb2:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732486426439.f0252adfc27b54a76a05585f3f45bf90. 2024-11-24T22:13:49,916 ERROR [RS_FLUSH_OPERATIONS-regionserver/bd6592c37bb2:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] handler.RSProcedureHandler(58): pid=8 java.io.IOException: Unable to complete flush {ENCODED => f0252adfc27b54a76a05585f3f45bf90, NAME => 'TestAcidGuarantees,,1732486426439.f0252adfc27b54a76a05585f3f45bf90.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T22:13:49,917 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bd6592c37bb2:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=8 java.io.IOException: Unable to complete flush {ENCODED => f0252adfc27b54a76a05585f3f45bf90, NAME => 'TestAcidGuarantees,,1732486426439.f0252adfc27b54a76a05585f3f45bf90.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T22:13:49,918 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK f0252adfc27b54a76a05585f3f45bf90, store=A 2024-11-24T22:13:49,918 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-24T22:13:49,918 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK f0252adfc27b54a76a05585f3f45bf90, store=B 2024-11-24T22:13:49,918 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-24T22:13:49,918 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK f0252adfc27b54a76a05585f3f45bf90, store=C 2024-11-24T22:13:49,918 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37701 {}] master.HMaster(4177): Remote procedure failed, pid=8 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => f0252adfc27b54a76a05585f3f45bf90, NAME => 'TestAcidGuarantees,,1732486426439.f0252adfc27b54a76a05585f3f45bf90.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => f0252adfc27b54a76a05585f3f45bf90, NAME => 'TestAcidGuarantees,,1732486426439.f0252adfc27b54a76a05585f3f45bf90.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T22:13:49,918 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-24T22:13:49,944 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/f0252adfc27b54a76a05585f3f45bf90/.tmp/A/f86308fe4a9f4a84a5e46be6687402f0 is 50, key is test_row_0/A:col10/1732486429896/Put/seqid=0 2024-11-24T22:13:49,973 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40087 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0252adfc27b54a76a05585f3f45bf90, server=bd6592c37bb2,40087,1732486423713 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-24T22:13:49,975 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40087 {}] ipc.CallRunner(138): callId: 36 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36562 deadline: 1732486489967, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0252adfc27b54a76a05585f3f45bf90, server=bd6592c37bb2,40087,1732486423713 2024-11-24T22:13:49,975 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45929 is added to blk_1073741843_1019 (size=19021) 2024-11-24T22:13:49,976 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732486426439.f0252adfc27b54a76a05585f3f45bf90., hostname=bd6592c37bb2,40087,1732486423713, seqNum=2 , the old value is region=TestAcidGuarantees,,1732486426439.f0252adfc27b54a76a05585f3f45bf90., hostname=bd6592c37bb2,40087,1732486423713, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0252adfc27b54a76a05585f3f45bf90, server=bd6592c37bb2,40087,1732486423713 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T22:13:49,977 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732486426439.f0252adfc27b54a76a05585f3f45bf90., hostname=bd6592c37bb2,40087,1732486423713, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0252adfc27b54a76a05585f3f45bf90, server=bd6592c37bb2,40087,1732486423713 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T22:13:49,977 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732486426439.f0252adfc27b54a76a05585f3f45bf90., hostname=bd6592c37bb2,40087,1732486423713, seqNum=2 because the exception is null or not the one we care about 2024-11-24T22:13:49,987 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40087 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0252adfc27b54a76a05585f3f45bf90, server=bd6592c37bb2,40087,1732486423713 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-24T22:13:49,988 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40087 {}] ipc.CallRunner(138): callId: 36 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36518 deadline: 1732486489972, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0252adfc27b54a76a05585f3f45bf90, server=bd6592c37bb2,40087,1732486423713 2024-11-24T22:13:49,989 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732486426439.f0252adfc27b54a76a05585f3f45bf90., hostname=bd6592c37bb2,40087,1732486423713, seqNum=2 , the old value is region=TestAcidGuarantees,,1732486426439.f0252adfc27b54a76a05585f3f45bf90., hostname=bd6592c37bb2,40087,1732486423713, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0252adfc27b54a76a05585f3f45bf90, server=bd6592c37bb2,40087,1732486423713 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T22:13:49,989 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732486426439.f0252adfc27b54a76a05585f3f45bf90., hostname=bd6592c37bb2,40087,1732486423713, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0252adfc27b54a76a05585f3f45bf90, server=bd6592c37bb2,40087,1732486423713 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T22:13:49,989 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732486426439.f0252adfc27b54a76a05585f3f45bf90., hostname=bd6592c37bb2,40087,1732486423713, seqNum=2 because the exception is null or not the one we care about 2024-11-24T22:13:49,990 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40087 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0252adfc27b54a76a05585f3f45bf90, server=bd6592c37bb2,40087,1732486423713 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-24T22:13:49,991 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40087 {}] ipc.CallRunner(138): callId: 37 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36532 deadline: 1732486489973, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0252adfc27b54a76a05585f3f45bf90, server=bd6592c37bb2,40087,1732486423713 2024-11-24T22:13:49,991 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732486426439.f0252adfc27b54a76a05585f3f45bf90., hostname=bd6592c37bb2,40087,1732486423713, seqNum=2 , the old value is region=TestAcidGuarantees,,1732486426439.f0252adfc27b54a76a05585f3f45bf90., hostname=bd6592c37bb2,40087,1732486423713, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0252adfc27b54a76a05585f3f45bf90, server=bd6592c37bb2,40087,1732486423713 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T22:13:49,992 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732486426439.f0252adfc27b54a76a05585f3f45bf90., hostname=bd6592c37bb2,40087,1732486423713, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0252adfc27b54a76a05585f3f45bf90, server=bd6592c37bb2,40087,1732486423713 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T22:13:49,992 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732486426439.f0252adfc27b54a76a05585f3f45bf90., hostname=bd6592c37bb2,40087,1732486423713, seqNum=2 because the exception is null or not the one we care about 2024-11-24T22:13:49,993 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40087 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0252adfc27b54a76a05585f3f45bf90, server=bd6592c37bb2,40087,1732486423713 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-24T22:13:49,993 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40087 {}] ipc.CallRunner(138): callId: 21 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36590 deadline: 1732486489975, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0252adfc27b54a76a05585f3f45bf90, server=bd6592c37bb2,40087,1732486423713 2024-11-24T22:13:49,994 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732486426439.f0252adfc27b54a76a05585f3f45bf90., hostname=bd6592c37bb2,40087,1732486423713, seqNum=2 , the old value is region=TestAcidGuarantees,,1732486426439.f0252adfc27b54a76a05585f3f45bf90., hostname=bd6592c37bb2,40087,1732486423713, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0252adfc27b54a76a05585f3f45bf90, server=bd6592c37bb2,40087,1732486423713 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T22:13:49,994 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732486426439.f0252adfc27b54a76a05585f3f45bf90., hostname=bd6592c37bb2,40087,1732486423713, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0252adfc27b54a76a05585f3f45bf90, server=bd6592c37bb2,40087,1732486423713 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T22:13:49,994 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732486426439.f0252adfc27b54a76a05585f3f45bf90., hostname=bd6592c37bb2,40087,1732486423713, seqNum=2 because the exception is null or not the one we care about 2024-11-24T22:13:50,072 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=40087 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=8 2024-11-24T22:13:50,072 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bd6592c37bb2:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732486426439.f0252adfc27b54a76a05585f3f45bf90. 2024-11-24T22:13:50,072 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bd6592c37bb2:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732486426439.f0252adfc27b54a76a05585f3f45bf90. as already flushing 2024-11-24T22:13:50,072 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bd6592c37bb2:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732486426439.f0252adfc27b54a76a05585f3f45bf90. 2024-11-24T22:13:50,072 ERROR [RS_FLUSH_OPERATIONS-regionserver/bd6592c37bb2:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] handler.RSProcedureHandler(58): pid=8 java.io.IOException: Unable to complete flush {ENCODED => f0252adfc27b54a76a05585f3f45bf90, NAME => 'TestAcidGuarantees,,1732486426439.f0252adfc27b54a76a05585f3f45bf90.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T22:13:50,073 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bd6592c37bb2:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=8 java.io.IOException: Unable to complete flush {ENCODED => f0252adfc27b54a76a05585f3f45bf90, NAME => 'TestAcidGuarantees,,1732486426439.f0252adfc27b54a76a05585f3f45bf90.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T22:13:50,074 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37701 {}] master.HMaster(4177): Remote procedure failed, pid=8 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => f0252adfc27b54a76a05585f3f45bf90, NAME => 'TestAcidGuarantees,,1732486426439.f0252adfc27b54a76a05585f3f45bf90.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => f0252adfc27b54a76a05585f3f45bf90, NAME => 'TestAcidGuarantees,,1732486426439.f0252adfc27b54a76a05585f3f45bf90.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T22:13:50,086 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40087 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0252adfc27b54a76a05585f3f45bf90, server=bd6592c37bb2,40087,1732486423713 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-24T22:13:50,087 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40087 {}] ipc.CallRunner(138): callId: 37 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36562 deadline: 1732486490084, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0252adfc27b54a76a05585f3f45bf90, server=bd6592c37bb2,40087,1732486423713 2024-11-24T22:13:50,088 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732486426439.f0252adfc27b54a76a05585f3f45bf90., hostname=bd6592c37bb2,40087,1732486423713, seqNum=2 , the old value is region=TestAcidGuarantees,,1732486426439.f0252adfc27b54a76a05585f3f45bf90., hostname=bd6592c37bb2,40087,1732486423713, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0252adfc27b54a76a05585f3f45bf90, server=bd6592c37bb2,40087,1732486423713 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T22:13:50,088 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732486426439.f0252adfc27b54a76a05585f3f45bf90., hostname=bd6592c37bb2,40087,1732486423713, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0252adfc27b54a76a05585f3f45bf90, server=bd6592c37bb2,40087,1732486423713 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T22:13:50,088 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732486426439.f0252adfc27b54a76a05585f3f45bf90., hostname=bd6592c37bb2,40087,1732486423713, seqNum=2 because the exception is null or not the one we care about 2024-11-24T22:13:50,106 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40087 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0252adfc27b54a76a05585f3f45bf90, server=bd6592c37bb2,40087,1732486423713 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-24T22:13:50,107 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40087 {}] ipc.CallRunner(138): callId: 37 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36518 deadline: 1732486490102, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0252adfc27b54a76a05585f3f45bf90, server=bd6592c37bb2,40087,1732486423713 2024-11-24T22:13:50,108 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40087 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0252adfc27b54a76a05585f3f45bf90, server=bd6592c37bb2,40087,1732486423713 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-24T22:13:50,108 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40087 {}] ipc.CallRunner(138): callId: 22 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36590 deadline: 1732486490103, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0252adfc27b54a76a05585f3f45bf90, server=bd6592c37bb2,40087,1732486423713 2024-11-24T22:13:50,109 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40087 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0252adfc27b54a76a05585f3f45bf90, server=bd6592c37bb2,40087,1732486423713 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-24T22:13:50,110 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40087 {}] ipc.CallRunner(138): callId: 38 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36532 deadline: 1732486490103, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0252adfc27b54a76a05585f3f45bf90, server=bd6592c37bb2,40087,1732486423713 2024-11-24T22:13:50,110 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732486426439.f0252adfc27b54a76a05585f3f45bf90., hostname=bd6592c37bb2,40087,1732486423713, seqNum=2 , the old value is region=TestAcidGuarantees,,1732486426439.f0252adfc27b54a76a05585f3f45bf90., hostname=bd6592c37bb2,40087,1732486423713, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0252adfc27b54a76a05585f3f45bf90, server=bd6592c37bb2,40087,1732486423713 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T22:13:50,110 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732486426439.f0252adfc27b54a76a05585f3f45bf90., hostname=bd6592c37bb2,40087,1732486423713, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0252adfc27b54a76a05585f3f45bf90, server=bd6592c37bb2,40087,1732486423713 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T22:13:50,111 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732486426439.f0252adfc27b54a76a05585f3f45bf90., hostname=bd6592c37bb2,40087,1732486423713, seqNum=2 because the exception is null or not the one we care about 2024-11-24T22:13:50,111 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732486426439.f0252adfc27b54a76a05585f3f45bf90., hostname=bd6592c37bb2,40087,1732486423713, seqNum=2 , the old value is region=TestAcidGuarantees,,1732486426439.f0252adfc27b54a76a05585f3f45bf90., hostname=bd6592c37bb2,40087,1732486423713, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0252adfc27b54a76a05585f3f45bf90, server=bd6592c37bb2,40087,1732486423713 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T22:13:50,111 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732486426439.f0252adfc27b54a76a05585f3f45bf90., hostname=bd6592c37bb2,40087,1732486423713, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0252adfc27b54a76a05585f3f45bf90, server=bd6592c37bb2,40087,1732486423713 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T22:13:50,111 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732486426439.f0252adfc27b54a76a05585f3f45bf90., hostname=bd6592c37bb2,40087,1732486423713, seqNum=2 because the exception is null or not the one we care about 2024-11-24T22:13:50,111 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732486426439.f0252adfc27b54a76a05585f3f45bf90., hostname=bd6592c37bb2,40087,1732486423713, seqNum=2 , the old value is region=TestAcidGuarantees,,1732486426439.f0252adfc27b54a76a05585f3f45bf90., hostname=bd6592c37bb2,40087,1732486423713, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0252adfc27b54a76a05585f3f45bf90, server=bd6592c37bb2,40087,1732486423713 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T22:13:50,112 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732486426439.f0252adfc27b54a76a05585f3f45bf90., hostname=bd6592c37bb2,40087,1732486423713, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0252adfc27b54a76a05585f3f45bf90, server=bd6592c37bb2,40087,1732486423713 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T22:13:50,112 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732486426439.f0252adfc27b54a76a05585f3f45bf90., hostname=bd6592c37bb2,40087,1732486423713, seqNum=2 because the exception is null or not the one we care about 2024-11-24T22:13:50,128 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40087 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0252adfc27b54a76a05585f3f45bf90, server=bd6592c37bb2,40087,1732486423713 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-24T22:13:50,129 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40087 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36548 deadline: 1732486490121, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0252adfc27b54a76a05585f3f45bf90, server=bd6592c37bb2,40087,1732486423713 2024-11-24T22:13:50,130 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732486426439.f0252adfc27b54a76a05585f3f45bf90., hostname=bd6592c37bb2,40087,1732486423713, seqNum=2 , the old value is region=TestAcidGuarantees,,1732486426439.f0252adfc27b54a76a05585f3f45bf90., hostname=bd6592c37bb2,40087,1732486423713, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0252adfc27b54a76a05585f3f45bf90, server=bd6592c37bb2,40087,1732486423713 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T22:13:50,130 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732486426439.f0252adfc27b54a76a05585f3f45bf90., hostname=bd6592c37bb2,40087,1732486423713, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0252adfc27b54a76a05585f3f45bf90, server=bd6592c37bb2,40087,1732486423713 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T22:13:50,130 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732486426439.f0252adfc27b54a76a05585f3f45bf90., hostname=bd6592c37bb2,40087,1732486423713, seqNum=2 because the exception is null or not the one we care about 2024-11-24T22:13:50,229 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=40087 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=8 2024-11-24T22:13:50,229 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bd6592c37bb2:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732486426439.f0252adfc27b54a76a05585f3f45bf90. 2024-11-24T22:13:50,229 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bd6592c37bb2:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732486426439.f0252adfc27b54a76a05585f3f45bf90. as already flushing 2024-11-24T22:13:50,230 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bd6592c37bb2:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732486426439.f0252adfc27b54a76a05585f3f45bf90. 2024-11-24T22:13:50,230 ERROR [RS_FLUSH_OPERATIONS-regionserver/bd6592c37bb2:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] handler.RSProcedureHandler(58): pid=8 java.io.IOException: Unable to complete flush {ENCODED => f0252adfc27b54a76a05585f3f45bf90, NAME => 'TestAcidGuarantees,,1732486426439.f0252adfc27b54a76a05585f3f45bf90.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T22:13:50,230 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bd6592c37bb2:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=8 java.io.IOException: Unable to complete flush {ENCODED => f0252adfc27b54a76a05585f3f45bf90, NAME => 'TestAcidGuarantees,,1732486426439.f0252adfc27b54a76a05585f3f45bf90.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T22:13:50,231 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37701 {}] master.HMaster(4177): Remote procedure failed, pid=8 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => f0252adfc27b54a76a05585f3f45bf90, NAME => 'TestAcidGuarantees,,1732486426439.f0252adfc27b54a76a05585f3f45bf90.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => f0252adfc27b54a76a05585f3f45bf90, NAME => 'TestAcidGuarantees,,1732486426439.f0252adfc27b54a76a05585f3f45bf90.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T22:13:50,297 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40087 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0252adfc27b54a76a05585f3f45bf90, server=bd6592c37bb2,40087,1732486423713 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-24T22:13:50,298 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40087 {}] ipc.CallRunner(138): callId: 38 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36562 deadline: 1732486490291, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0252adfc27b54a76a05585f3f45bf90, server=bd6592c37bb2,40087,1732486423713 2024-11-24T22:13:50,299 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732486426439.f0252adfc27b54a76a05585f3f45bf90., hostname=bd6592c37bb2,40087,1732486423713, seqNum=2 , the old value is region=TestAcidGuarantees,,1732486426439.f0252adfc27b54a76a05585f3f45bf90., hostname=bd6592c37bb2,40087,1732486423713, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0252adfc27b54a76a05585f3f45bf90, server=bd6592c37bb2,40087,1732486423713 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T22:13:50,299 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732486426439.f0252adfc27b54a76a05585f3f45bf90., hostname=bd6592c37bb2,40087,1732486423713, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0252adfc27b54a76a05585f3f45bf90, server=bd6592c37bb2,40087,1732486423713 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T22:13:50,299 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732486426439.f0252adfc27b54a76a05585f3f45bf90., hostname=bd6592c37bb2,40087,1732486423713, seqNum=2 because the exception is null or not the one we care about 2024-11-24T22:13:50,327 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40087 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0252adfc27b54a76a05585f3f45bf90, server=bd6592c37bb2,40087,1732486423713 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-24T22:13:50,328 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40087 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0252adfc27b54a76a05585f3f45bf90, server=bd6592c37bb2,40087,1732486423713 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-24T22:13:50,328 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40087 {}] ipc.CallRunner(138): callId: 38 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36518 deadline: 1732486490322, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0252adfc27b54a76a05585f3f45bf90, server=bd6592c37bb2,40087,1732486423713 2024-11-24T22:13:50,328 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40087 {}] ipc.CallRunner(138): callId: 39 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36532 deadline: 1732486490322, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0252adfc27b54a76a05585f3f45bf90, server=bd6592c37bb2,40087,1732486423713 2024-11-24T22:13:50,329 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732486426439.f0252adfc27b54a76a05585f3f45bf90., hostname=bd6592c37bb2,40087,1732486423713, seqNum=2 , the old value is region=TestAcidGuarantees,,1732486426439.f0252adfc27b54a76a05585f3f45bf90., hostname=bd6592c37bb2,40087,1732486423713, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0252adfc27b54a76a05585f3f45bf90, server=bd6592c37bb2,40087,1732486423713 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T22:13:50,329 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732486426439.f0252adfc27b54a76a05585f3f45bf90., hostname=bd6592c37bb2,40087,1732486423713, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0252adfc27b54a76a05585f3f45bf90, server=bd6592c37bb2,40087,1732486423713 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T22:13:50,329 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40087 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0252adfc27b54a76a05585f3f45bf90, server=bd6592c37bb2,40087,1732486423713 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-24T22:13:50,329 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732486426439.f0252adfc27b54a76a05585f3f45bf90., hostname=bd6592c37bb2,40087,1732486423713, seqNum=2 because the exception is null or not the one we care about 2024-11-24T22:13:50,329 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40087 {}] ipc.CallRunner(138): callId: 23 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36590 deadline: 1732486490323, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0252adfc27b54a76a05585f3f45bf90, server=bd6592c37bb2,40087,1732486423713 2024-11-24T22:13:50,337 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732486426439.f0252adfc27b54a76a05585f3f45bf90., hostname=bd6592c37bb2,40087,1732486423713, seqNum=2 , the old value is region=TestAcidGuarantees,,1732486426439.f0252adfc27b54a76a05585f3f45bf90., hostname=bd6592c37bb2,40087,1732486423713, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0252adfc27b54a76a05585f3f45bf90, server=bd6592c37bb2,40087,1732486423713 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T22:13:50,337 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732486426439.f0252adfc27b54a76a05585f3f45bf90., hostname=bd6592c37bb2,40087,1732486423713, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0252adfc27b54a76a05585f3f45bf90, server=bd6592c37bb2,40087,1732486423713 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T22:13:50,337 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732486426439.f0252adfc27b54a76a05585f3f45bf90., hostname=bd6592c37bb2,40087,1732486423713, seqNum=2 because the exception is null or not the one we care about 2024-11-24T22:13:50,338 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732486426439.f0252adfc27b54a76a05585f3f45bf90., hostname=bd6592c37bb2,40087,1732486423713, seqNum=2 , the old value is region=TestAcidGuarantees,,1732486426439.f0252adfc27b54a76a05585f3f45bf90., hostname=bd6592c37bb2,40087,1732486423713, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0252adfc27b54a76a05585f3f45bf90, server=bd6592c37bb2,40087,1732486423713 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T22:13:50,338 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732486426439.f0252adfc27b54a76a05585f3f45bf90., hostname=bd6592c37bb2,40087,1732486423713, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0252adfc27b54a76a05585f3f45bf90, server=bd6592c37bb2,40087,1732486423713 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T22:13:50,338 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732486426439.f0252adfc27b54a76a05585f3f45bf90., hostname=bd6592c37bb2,40087,1732486423713, seqNum=2 because the exception is null or not the one we care about 2024-11-24T22:13:50,377 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=35.78 KB at sequenceid=120 (bloomFilter=true), to=hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/f0252adfc27b54a76a05585f3f45bf90/.tmp/A/f86308fe4a9f4a84a5e46be6687402f0 2024-11-24T22:13:50,387 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=40087 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=8 2024-11-24T22:13:50,388 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bd6592c37bb2:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732486426439.f0252adfc27b54a76a05585f3f45bf90. 2024-11-24T22:13:50,388 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bd6592c37bb2:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732486426439.f0252adfc27b54a76a05585f3f45bf90. as already flushing 2024-11-24T22:13:50,388 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bd6592c37bb2:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732486426439.f0252adfc27b54a76a05585f3f45bf90. 2024-11-24T22:13:50,388 ERROR [RS_FLUSH_OPERATIONS-regionserver/bd6592c37bb2:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] handler.RSProcedureHandler(58): pid=8 java.io.IOException: Unable to complete flush {ENCODED => f0252adfc27b54a76a05585f3f45bf90, NAME => 'TestAcidGuarantees,,1732486426439.f0252adfc27b54a76a05585f3f45bf90.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T22:13:50,388 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bd6592c37bb2:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=8 java.io.IOException: Unable to complete flush {ENCODED => f0252adfc27b54a76a05585f3f45bf90, NAME => 'TestAcidGuarantees,,1732486426439.f0252adfc27b54a76a05585f3f45bf90.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T22:13:50,390 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37701 {}] master.HMaster(4177): Remote procedure failed, pid=8 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => f0252adfc27b54a76a05585f3f45bf90, NAME => 'TestAcidGuarantees,,1732486426439.f0252adfc27b54a76a05585f3f45bf90.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => f0252adfc27b54a76a05585f3f45bf90, NAME => 'TestAcidGuarantees,,1732486426439.f0252adfc27b54a76a05585f3f45bf90.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T22:13:50,412 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/f0252adfc27b54a76a05585f3f45bf90/.tmp/B/5609bc3ed6cb49db83278dc1722a8dbb is 50, key is test_row_0/B:col10/1732486429896/Put/seqid=0 2024-11-24T22:13:50,447 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45929 is added to blk_1073741844_1020 (size=12001) 2024-11-24T22:13:50,544 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=40087 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=8 2024-11-24T22:13:50,544 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bd6592c37bb2:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732486426439.f0252adfc27b54a76a05585f3f45bf90. 2024-11-24T22:13:50,544 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bd6592c37bb2:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732486426439.f0252adfc27b54a76a05585f3f45bf90. as already flushing 2024-11-24T22:13:50,545 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bd6592c37bb2:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732486426439.f0252adfc27b54a76a05585f3f45bf90. 2024-11-24T22:13:50,545 ERROR [RS_FLUSH_OPERATIONS-regionserver/bd6592c37bb2:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] handler.RSProcedureHandler(58): pid=8 java.io.IOException: Unable to complete flush {ENCODED => f0252adfc27b54a76a05585f3f45bf90, NAME => 'TestAcidGuarantees,,1732486426439.f0252adfc27b54a76a05585f3f45bf90.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T22:13:50,545 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bd6592c37bb2:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=8 java.io.IOException: Unable to complete flush {ENCODED => f0252adfc27b54a76a05585f3f45bf90, NAME => 'TestAcidGuarantees,,1732486426439.f0252adfc27b54a76a05585f3f45bf90.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T22:13:50,546 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37701 {}] master.HMaster(4177): Remote procedure failed, pid=8 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => f0252adfc27b54a76a05585f3f45bf90, NAME => 'TestAcidGuarantees,,1732486426439.f0252adfc27b54a76a05585f3f45bf90.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => f0252adfc27b54a76a05585f3f45bf90, NAME => 'TestAcidGuarantees,,1732486426439.f0252adfc27b54a76a05585f3f45bf90.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T22:13:50,608 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40087 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0252adfc27b54a76a05585f3f45bf90, server=bd6592c37bb2,40087,1732486423713 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-24T22:13:50,608 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40087 {}] ipc.CallRunner(138): callId: 39 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36562 deadline: 1732486490601, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0252adfc27b54a76a05585f3f45bf90, server=bd6592c37bb2,40087,1732486423713 2024-11-24T22:13:50,609 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732486426439.f0252adfc27b54a76a05585f3f45bf90., hostname=bd6592c37bb2,40087,1732486423713, seqNum=2 , the old value is region=TestAcidGuarantees,,1732486426439.f0252adfc27b54a76a05585f3f45bf90., hostname=bd6592c37bb2,40087,1732486423713, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0252adfc27b54a76a05585f3f45bf90, server=bd6592c37bb2,40087,1732486423713 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T22:13:50,609 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732486426439.f0252adfc27b54a76a05585f3f45bf90., hostname=bd6592c37bb2,40087,1732486423713, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0252adfc27b54a76a05585f3f45bf90, server=bd6592c37bb2,40087,1732486423713 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T22:13:50,609 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732486426439.f0252adfc27b54a76a05585f3f45bf90., hostname=bd6592c37bb2,40087,1732486423713, seqNum=2 because the exception is null or not the one we care about 2024-11-24T22:13:50,646 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40087 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0252adfc27b54a76a05585f3f45bf90, server=bd6592c37bb2,40087,1732486423713 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-24T22:13:50,646 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40087 {}] ipc.CallRunner(138): callId: 39 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36518 deadline: 1732486490641, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0252adfc27b54a76a05585f3f45bf90, server=bd6592c37bb2,40087,1732486423713 2024-11-24T22:13:50,647 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732486426439.f0252adfc27b54a76a05585f3f45bf90., hostname=bd6592c37bb2,40087,1732486423713, seqNum=2 , the old value is region=TestAcidGuarantees,,1732486426439.f0252adfc27b54a76a05585f3f45bf90., hostname=bd6592c37bb2,40087,1732486423713, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0252adfc27b54a76a05585f3f45bf90, server=bd6592c37bb2,40087,1732486423713 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T22:13:50,647 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40087 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0252adfc27b54a76a05585f3f45bf90, server=bd6592c37bb2,40087,1732486423713 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-24T22:13:50,647 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732486426439.f0252adfc27b54a76a05585f3f45bf90., hostname=bd6592c37bb2,40087,1732486423713, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0252adfc27b54a76a05585f3f45bf90, server=bd6592c37bb2,40087,1732486423713 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T22:13:50,647 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732486426439.f0252adfc27b54a76a05585f3f45bf90., hostname=bd6592c37bb2,40087,1732486423713, seqNum=2 because the exception is null or not the one we care about 2024-11-24T22:13:50,648 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40087 {}] ipc.CallRunner(138): callId: 40 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36532 deadline: 1732486490642, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0252adfc27b54a76a05585f3f45bf90, server=bd6592c37bb2,40087,1732486423713 2024-11-24T22:13:50,648 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732486426439.f0252adfc27b54a76a05585f3f45bf90., hostname=bd6592c37bb2,40087,1732486423713, seqNum=2 , the old value is region=TestAcidGuarantees,,1732486426439.f0252adfc27b54a76a05585f3f45bf90., hostname=bd6592c37bb2,40087,1732486423713, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0252adfc27b54a76a05585f3f45bf90, server=bd6592c37bb2,40087,1732486423713 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T22:13:50,648 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732486426439.f0252adfc27b54a76a05585f3f45bf90., hostname=bd6592c37bb2,40087,1732486423713, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0252adfc27b54a76a05585f3f45bf90, server=bd6592c37bb2,40087,1732486423713 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T22:13:50,648 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40087 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0252adfc27b54a76a05585f3f45bf90, server=bd6592c37bb2,40087,1732486423713 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-24T22:13:50,648 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732486426439.f0252adfc27b54a76a05585f3f45bf90., hostname=bd6592c37bb2,40087,1732486423713, seqNum=2 because the exception is null or not the one we care about 2024-11-24T22:13:50,649 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40087 {}] ipc.CallRunner(138): callId: 24 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36590 deadline: 1732486490642, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0252adfc27b54a76a05585f3f45bf90, server=bd6592c37bb2,40087,1732486423713 2024-11-24T22:13:50,649 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732486426439.f0252adfc27b54a76a05585f3f45bf90., hostname=bd6592c37bb2,40087,1732486423713, seqNum=2 , the old value is region=TestAcidGuarantees,,1732486426439.f0252adfc27b54a76a05585f3f45bf90., hostname=bd6592c37bb2,40087,1732486423713, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0252adfc27b54a76a05585f3f45bf90, server=bd6592c37bb2,40087,1732486423713 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T22:13:50,649 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732486426439.f0252adfc27b54a76a05585f3f45bf90., hostname=bd6592c37bb2,40087,1732486423713, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0252adfc27b54a76a05585f3f45bf90, server=bd6592c37bb2,40087,1732486423713 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T22:13:50,649 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732486426439.f0252adfc27b54a76a05585f3f45bf90., hostname=bd6592c37bb2,40087,1732486423713, seqNum=2 because the exception is null or not the one we care about 2024-11-24T22:13:50,701 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=40087 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=8 2024-11-24T22:13:50,701 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bd6592c37bb2:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732486426439.f0252adfc27b54a76a05585f3f45bf90. 2024-11-24T22:13:50,701 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bd6592c37bb2:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732486426439.f0252adfc27b54a76a05585f3f45bf90. as already flushing 2024-11-24T22:13:50,701 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bd6592c37bb2:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732486426439.f0252adfc27b54a76a05585f3f45bf90. 2024-11-24T22:13:50,701 ERROR [RS_FLUSH_OPERATIONS-regionserver/bd6592c37bb2:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] handler.RSProcedureHandler(58): pid=8 java.io.IOException: Unable to complete flush {ENCODED => f0252adfc27b54a76a05585f3f45bf90, NAME => 'TestAcidGuarantees,,1732486426439.f0252adfc27b54a76a05585f3f45bf90.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T22:13:50,702 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bd6592c37bb2:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=8 java.io.IOException: Unable to complete flush {ENCODED => f0252adfc27b54a76a05585f3f45bf90, NAME => 'TestAcidGuarantees,,1732486426439.f0252adfc27b54a76a05585f3f45bf90.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T22:13:50,703 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37701 {}] master.HMaster(4177): Remote procedure failed, pid=8 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => f0252adfc27b54a76a05585f3f45bf90, NAME => 'TestAcidGuarantees,,1732486426439.f0252adfc27b54a76a05585f3f45bf90.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => f0252adfc27b54a76a05585f3f45bf90, NAME => 'TestAcidGuarantees,,1732486426439.f0252adfc27b54a76a05585f3f45bf90.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T22:13:50,848 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=35.78 KB at sequenceid=120 (bloomFilter=true), to=hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/f0252adfc27b54a76a05585f3f45bf90/.tmp/B/5609bc3ed6cb49db83278dc1722a8dbb 2024-11-24T22:13:50,857 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=40087 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=8 2024-11-24T22:13:50,857 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bd6592c37bb2:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732486426439.f0252adfc27b54a76a05585f3f45bf90. 2024-11-24T22:13:50,857 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bd6592c37bb2:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732486426439.f0252adfc27b54a76a05585f3f45bf90. as already flushing 2024-11-24T22:13:50,857 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bd6592c37bb2:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732486426439.f0252adfc27b54a76a05585f3f45bf90. 2024-11-24T22:13:50,858 ERROR [RS_FLUSH_OPERATIONS-regionserver/bd6592c37bb2:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] handler.RSProcedureHandler(58): pid=8 java.io.IOException: Unable to complete flush {ENCODED => f0252adfc27b54a76a05585f3f45bf90, NAME => 'TestAcidGuarantees,,1732486426439.f0252adfc27b54a76a05585f3f45bf90.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T22:13:50,858 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bd6592c37bb2:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=8 java.io.IOException: Unable to complete flush {ENCODED => f0252adfc27b54a76a05585f3f45bf90, NAME => 'TestAcidGuarantees,,1732486426439.f0252adfc27b54a76a05585f3f45bf90.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T22:13:50,859 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37701 {}] master.HMaster(4177): Remote procedure failed, pid=8 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => f0252adfc27b54a76a05585f3f45bf90, NAME => 'TestAcidGuarantees,,1732486426439.f0252adfc27b54a76a05585f3f45bf90.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => f0252adfc27b54a76a05585f3f45bf90, NAME => 'TestAcidGuarantees,,1732486426439.f0252adfc27b54a76a05585f3f45bf90.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T22:13:50,871 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37701 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=7 2024-11-24T22:13:50,875 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/f0252adfc27b54a76a05585f3f45bf90/.tmp/C/c3390572911442069a3598a2de63958d is 50, key is test_row_0/C:col10/1732486429896/Put/seqid=0 2024-11-24T22:13:50,891 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45929 is added to blk_1073741845_1021 (size=12001) 2024-11-24T22:13:50,892 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=35.78 KB at sequenceid=120 (bloomFilter=true), to=hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/f0252adfc27b54a76a05585f3f45bf90/.tmp/C/c3390572911442069a3598a2de63958d 2024-11-24T22:13:50,907 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/f0252adfc27b54a76a05585f3f45bf90/.tmp/A/f86308fe4a9f4a84a5e46be6687402f0 as hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/f0252adfc27b54a76a05585f3f45bf90/A/f86308fe4a9f4a84a5e46be6687402f0 2024-11-24T22:13:50,924 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/f0252adfc27b54a76a05585f3f45bf90/A/f86308fe4a9f4a84a5e46be6687402f0, entries=300, sequenceid=120, filesize=18.6 K 2024-11-24T22:13:50,927 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/f0252adfc27b54a76a05585f3f45bf90/.tmp/B/5609bc3ed6cb49db83278dc1722a8dbb as hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/f0252adfc27b54a76a05585f3f45bf90/B/5609bc3ed6cb49db83278dc1722a8dbb 2024-11-24T22:13:50,938 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/f0252adfc27b54a76a05585f3f45bf90/B/5609bc3ed6cb49db83278dc1722a8dbb, entries=150, sequenceid=120, filesize=11.7 K 2024-11-24T22:13:50,942 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/f0252adfc27b54a76a05585f3f45bf90/.tmp/C/c3390572911442069a3598a2de63958d as hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/f0252adfc27b54a76a05585f3f45bf90/C/c3390572911442069a3598a2de63958d 2024-11-24T22:13:50,955 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/f0252adfc27b54a76a05585f3f45bf90/C/c3390572911442069a3598a2de63958d, entries=150, sequenceid=120, filesize=11.7 K 2024-11-24T22:13:50,956 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~107.34 KB/109920, heapSize ~298.03 KB/305184, currentSize=73.80 KB/75570 for f0252adfc27b54a76a05585f3f45bf90 in 1041ms, sequenceid=120, compaction requested=true 2024-11-24T22:13:50,957 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for f0252adfc27b54a76a05585f3f45bf90: 2024-11-24T22:13:50,959 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store f0252adfc27b54a76a05585f3f45bf90:A, priority=-2147483648, current under compaction store size is 1 2024-11-24T22:13:50,962 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-24T22:13:50,962 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store f0252adfc27b54a76a05585f3f45bf90:B, priority=-2147483648, current under compaction store size is 2 2024-11-24T22:13:50,962 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-24T22:13:50,962 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store f0252adfc27b54a76a05585f3f45bf90:C, priority=-2147483648, current under compaction store size is 3 2024-11-24T22:13:50,962 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-24T22:13:50,963 DEBUG [RS:0;bd6592c37bb2:40087-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-24T22:13:50,964 DEBUG [RS:0;bd6592c37bb2:40087-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-24T22:13:50,967 DEBUG [RS:0;bd6592c37bb2:40087-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36003 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-24T22:13:50,969 DEBUG [RS:0;bd6592c37bb2:40087-longCompactions-0 {}] regionserver.HStore(1541): f0252adfc27b54a76a05585f3f45bf90/B is initiating minor compaction (all files) 2024-11-24T22:13:50,970 INFO [RS:0;bd6592c37bb2:40087-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of f0252adfc27b54a76a05585f3f45bf90/B in TestAcidGuarantees,,1732486426439.f0252adfc27b54a76a05585f3f45bf90. 2024-11-24T22:13:50,971 INFO [RS:0;bd6592c37bb2:40087-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/f0252adfc27b54a76a05585f3f45bf90/B/fa858762490548298016038bb1fe3609, hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/f0252adfc27b54a76a05585f3f45bf90/B/b7e5d8cf64e84135afbc9fb3af466717, hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/f0252adfc27b54a76a05585f3f45bf90/B/5609bc3ed6cb49db83278dc1722a8dbb] into tmpdir=hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/f0252adfc27b54a76a05585f3f45bf90/.tmp, totalSize=35.2 K 2024-11-24T22:13:50,972 DEBUG [RS:0;bd6592c37bb2:40087-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 52383 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-24T22:13:50,972 DEBUG [RS:0;bd6592c37bb2:40087-shortCompactions-0 {}] regionserver.HStore(1541): f0252adfc27b54a76a05585f3f45bf90/A is initiating minor compaction (all files) 2024-11-24T22:13:50,972 DEBUG [RS:0;bd6592c37bb2:40087-longCompactions-0 {}] compactions.Compactor(225): Compacting fa858762490548298016038bb1fe3609, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=14, earliestPutTs=1732486428735 2024-11-24T22:13:50,972 INFO [RS:0;bd6592c37bb2:40087-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of f0252adfc27b54a76a05585f3f45bf90/A in TestAcidGuarantees,,1732486426439.f0252adfc27b54a76a05585f3f45bf90. 2024-11-24T22:13:50,973 INFO [RS:0;bd6592c37bb2:40087-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/f0252adfc27b54a76a05585f3f45bf90/A/85bc3afcd03143c1b1f0def65f0e2e63, hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/f0252adfc27b54a76a05585f3f45bf90/A/c148305567ff449293163aa4d9ca6086, hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/f0252adfc27b54a76a05585f3f45bf90/A/f86308fe4a9f4a84a5e46be6687402f0] into tmpdir=hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/f0252adfc27b54a76a05585f3f45bf90/.tmp, totalSize=51.2 K 2024-11-24T22:13:50,975 DEBUG [RS:0;bd6592c37bb2:40087-shortCompactions-0 {}] compactions.Compactor(225): Compacting 85bc3afcd03143c1b1f0def65f0e2e63, keycount=200, bloomtype=ROW, size=14.0 K, encoding=NONE, compression=NONE, seqNum=14, earliestPutTs=1732486428735 2024-11-24T22:13:50,978 DEBUG [RS:0;bd6592c37bb2:40087-longCompactions-0 {}] compactions.Compactor(225): Compacting b7e5d8cf64e84135afbc9fb3af466717, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=37, earliestPutTs=1732486428906 2024-11-24T22:13:50,981 DEBUG [RS:0;bd6592c37bb2:40087-longCompactions-0 {}] compactions.Compactor(225): Compacting 5609bc3ed6cb49db83278dc1722a8dbb, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=120, earliestPutTs=1732486429896 2024-11-24T22:13:50,984 DEBUG [RS:0;bd6592c37bb2:40087-shortCompactions-0 {}] compactions.Compactor(225): Compacting c148305567ff449293163aa4d9ca6086, keycount=300, bloomtype=ROW, size=18.6 K, encoding=NONE, compression=NONE, seqNum=37, earliestPutTs=1732486428890 2024-11-24T22:13:50,985 DEBUG [RS:0;bd6592c37bb2:40087-shortCompactions-0 {}] compactions.Compactor(225): Compacting f86308fe4a9f4a84a5e46be6687402f0, keycount=300, bloomtype=ROW, size=18.6 K, encoding=NONE, compression=NONE, seqNum=120, earliestPutTs=1732486429853 2024-11-24T22:13:51,016 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=40087 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=8 2024-11-24T22:13:51,016 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bd6592c37bb2:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732486426439.f0252adfc27b54a76a05585f3f45bf90. 2024-11-24T22:13:51,017 INFO [RS_FLUSH_OPERATIONS-regionserver/bd6592c37bb2:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.HRegion(2902): Flushing f0252adfc27b54a76a05585f3f45bf90 3/3 column families, dataSize=73.80 KB heapSize=219.89 KB 2024-11-24T22:13:51,017 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bd6592c37bb2:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.CompactingMemStore(205): FLUSHING TO DISK f0252adfc27b54a76a05585f3f45bf90, store=A 2024-11-24T22:13:51,017 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bd6592c37bb2:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-24T22:13:51,017 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bd6592c37bb2:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.CompactingMemStore(205): FLUSHING TO DISK f0252adfc27b54a76a05585f3f45bf90, store=B 2024-11-24T22:13:51,018 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bd6592c37bb2:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-24T22:13:51,018 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bd6592c37bb2:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.CompactingMemStore(205): FLUSHING TO DISK f0252adfc27b54a76a05585f3f45bf90, store=C 2024-11-24T22:13:51,018 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bd6592c37bb2:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-24T22:13:51,028 INFO [RS:0;bd6592c37bb2:40087-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): f0252adfc27b54a76a05585f3f45bf90#B#compaction#9 average throughput is 0.50 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-24T22:13:51,029 DEBUG [RS:0;bd6592c37bb2:40087-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/f0252adfc27b54a76a05585f3f45bf90/.tmp/B/6cdf2f3301bb413f9d1e2d9b0160fee3 is 50, key is test_row_0/B:col10/1732486429896/Put/seqid=0 2024-11-24T22:13:51,030 INFO [RS:0;bd6592c37bb2:40087-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): f0252adfc27b54a76a05585f3f45bf90#A#compaction#10 average throughput is 0.41 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-24T22:13:51,031 DEBUG [RS:0;bd6592c37bb2:40087-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/f0252adfc27b54a76a05585f3f45bf90/.tmp/A/0a2765b1e92043a9b4d3fd87edcc95cd is 50, key is test_row_0/A:col10/1732486429896/Put/seqid=0 2024-11-24T22:13:51,035 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bd6592c37bb2:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/f0252adfc27b54a76a05585f3f45bf90/.tmp/A/705ad699596346dba6cf6781c5011618 is 50, key is test_row_0/A:col10/1732486429971/Put/seqid=0 2024-11-24T22:13:51,062 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45929 is added to blk_1073741846_1022 (size=12104) 2024-11-24T22:13:51,073 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45929 is added to blk_1073741847_1023 (size=12104) 2024-11-24T22:13:51,091 DEBUG [RS:0;bd6592c37bb2:40087-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/f0252adfc27b54a76a05585f3f45bf90/.tmp/A/0a2765b1e92043a9b4d3fd87edcc95cd as hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/f0252adfc27b54a76a05585f3f45bf90/A/0a2765b1e92043a9b4d3fd87edcc95cd 2024-11-24T22:13:51,096 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45929 is added to blk_1073741848_1024 (size=12101) 2024-11-24T22:13:51,116 INFO [RS:0;bd6592c37bb2:40087-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in f0252adfc27b54a76a05585f3f45bf90/A of f0252adfc27b54a76a05585f3f45bf90 into 0a2765b1e92043a9b4d3fd87edcc95cd(size=11.8 K), total size for store is 11.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-24T22:13:51,116 DEBUG [RS:0;bd6592c37bb2:40087-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for f0252adfc27b54a76a05585f3f45bf90: 2024-11-24T22:13:51,119 INFO [RS:0;bd6592c37bb2:40087-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732486426439.f0252adfc27b54a76a05585f3f45bf90., storeName=f0252adfc27b54a76a05585f3f45bf90/A, priority=13, startTime=1732486430958; duration=0sec 2024-11-24T22:13:51,120 DEBUG [RS:0;bd6592c37bb2:40087-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-24T22:13:51,120 DEBUG [RS:0;bd6592c37bb2:40087-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: f0252adfc27b54a76a05585f3f45bf90:A 2024-11-24T22:13:51,120 DEBUG [RS:0;bd6592c37bb2:40087-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-24T22:13:51,123 DEBUG [RS:0;bd6592c37bb2:40087-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36003 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-24T22:13:51,123 DEBUG [RS:0;bd6592c37bb2:40087-shortCompactions-0 {}] regionserver.HStore(1541): f0252adfc27b54a76a05585f3f45bf90/C is initiating minor compaction (all files) 2024-11-24T22:13:51,123 INFO [RS:0;bd6592c37bb2:40087-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of f0252adfc27b54a76a05585f3f45bf90/C in TestAcidGuarantees,,1732486426439.f0252adfc27b54a76a05585f3f45bf90. 2024-11-24T22:13:51,123 INFO [RS:0;bd6592c37bb2:40087-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/f0252adfc27b54a76a05585f3f45bf90/C/6cb278bb60bd4562b947a68e3f06567b, hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/f0252adfc27b54a76a05585f3f45bf90/C/cb90eafdaeba493ea99a4ef7d255056f, hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/f0252adfc27b54a76a05585f3f45bf90/C/c3390572911442069a3598a2de63958d] into tmpdir=hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/f0252adfc27b54a76a05585f3f45bf90/.tmp, totalSize=35.2 K 2024-11-24T22:13:51,125 DEBUG [RS:0;bd6592c37bb2:40087-shortCompactions-0 {}] compactions.Compactor(225): Compacting 6cb278bb60bd4562b947a68e3f06567b, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=14, earliestPutTs=1732486428735 2024-11-24T22:13:51,126 DEBUG [RS:0;bd6592c37bb2:40087-shortCompactions-0 {}] compactions.Compactor(225): Compacting cb90eafdaeba493ea99a4ef7d255056f, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=37, earliestPutTs=1732486428906 2024-11-24T22:13:51,127 DEBUG [RS:0;bd6592c37bb2:40087-shortCompactions-0 {}] compactions.Compactor(225): Compacting c3390572911442069a3598a2de63958d, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=120, earliestPutTs=1732486429896 2024-11-24T22:13:51,128 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40087 {}] regionserver.HRegion(8855): Flush requested on f0252adfc27b54a76a05585f3f45bf90 2024-11-24T22:13:51,128 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732486426439.f0252adfc27b54a76a05585f3f45bf90. as already flushing 2024-11-24T22:13:51,151 INFO [RS:0;bd6592c37bb2:40087-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): f0252adfc27b54a76a05585f3f45bf90#C#compaction#12 average throughput is 1.64 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-24T22:13:51,152 DEBUG [RS:0;bd6592c37bb2:40087-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/f0252adfc27b54a76a05585f3f45bf90/.tmp/C/88c916dc5d6b43af9776f182ebf2959c is 50, key is test_row_0/C:col10/1732486429896/Put/seqid=0 2024-11-24T22:13:51,178 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45929 is added to blk_1073741849_1025 (size=12104) 2024-11-24T22:13:51,192 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40087 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0252adfc27b54a76a05585f3f45bf90, server=bd6592c37bb2,40087,1732486423713 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-24T22:13:51,193 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40087 {}] ipc.CallRunner(138): callId: 46 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36562 deadline: 1732486491180, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0252adfc27b54a76a05585f3f45bf90, server=bd6592c37bb2,40087,1732486423713 2024-11-24T22:13:51,194 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732486426439.f0252adfc27b54a76a05585f3f45bf90., hostname=bd6592c37bb2,40087,1732486423713, seqNum=2 , the old value is region=TestAcidGuarantees,,1732486426439.f0252adfc27b54a76a05585f3f45bf90., hostname=bd6592c37bb2,40087,1732486423713, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0252adfc27b54a76a05585f3f45bf90, server=bd6592c37bb2,40087,1732486423713 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T22:13:51,194 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40087 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0252adfc27b54a76a05585f3f45bf90, server=bd6592c37bb2,40087,1732486423713 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-24T22:13:51,194 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732486426439.f0252adfc27b54a76a05585f3f45bf90., hostname=bd6592c37bb2,40087,1732486423713, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0252adfc27b54a76a05585f3f45bf90, server=bd6592c37bb2,40087,1732486423713 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T22:13:51,194 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732486426439.f0252adfc27b54a76a05585f3f45bf90., hostname=bd6592c37bb2,40087,1732486423713, seqNum=2 because the exception is null or not the one we care about 2024-11-24T22:13:51,194 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40087 {}] ipc.CallRunner(138): callId: 26 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36590 deadline: 1732486491187, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0252adfc27b54a76a05585f3f45bf90, server=bd6592c37bb2,40087,1732486423713 2024-11-24T22:13:51,194 DEBUG [RS:0;bd6592c37bb2:40087-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/f0252adfc27b54a76a05585f3f45bf90/.tmp/C/88c916dc5d6b43af9776f182ebf2959c as hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/f0252adfc27b54a76a05585f3f45bf90/C/88c916dc5d6b43af9776f182ebf2959c 2024-11-24T22:13:51,195 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732486426439.f0252adfc27b54a76a05585f3f45bf90., hostname=bd6592c37bb2,40087,1732486423713, seqNum=2 , the old value is region=TestAcidGuarantees,,1732486426439.f0252adfc27b54a76a05585f3f45bf90., hostname=bd6592c37bb2,40087,1732486423713, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0252adfc27b54a76a05585f3f45bf90, server=bd6592c37bb2,40087,1732486423713 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T22:13:51,195 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40087 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0252adfc27b54a76a05585f3f45bf90, server=bd6592c37bb2,40087,1732486423713 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-24T22:13:51,195 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732486426439.f0252adfc27b54a76a05585f3f45bf90., hostname=bd6592c37bb2,40087,1732486423713, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0252adfc27b54a76a05585f3f45bf90, server=bd6592c37bb2,40087,1732486423713 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T22:13:51,195 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732486426439.f0252adfc27b54a76a05585f3f45bf90., hostname=bd6592c37bb2,40087,1732486423713, seqNum=2 because the exception is null or not the one we care about 2024-11-24T22:13:51,195 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40087 {}] ipc.CallRunner(138): callId: 42 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36532 deadline: 1732486491187, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0252adfc27b54a76a05585f3f45bf90, server=bd6592c37bb2,40087,1732486423713 2024-11-24T22:13:51,196 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732486426439.f0252adfc27b54a76a05585f3f45bf90., hostname=bd6592c37bb2,40087,1732486423713, seqNum=2 , the old value is region=TestAcidGuarantees,,1732486426439.f0252adfc27b54a76a05585f3f45bf90., hostname=bd6592c37bb2,40087,1732486423713, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0252adfc27b54a76a05585f3f45bf90, server=bd6592c37bb2,40087,1732486423713 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T22:13:51,196 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732486426439.f0252adfc27b54a76a05585f3f45bf90., hostname=bd6592c37bb2,40087,1732486423713, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0252adfc27b54a76a05585f3f45bf90, server=bd6592c37bb2,40087,1732486423713 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T22:13:51,196 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732486426439.f0252adfc27b54a76a05585f3f45bf90., hostname=bd6592c37bb2,40087,1732486423713, seqNum=2 because the exception is null or not the one we care about 2024-11-24T22:13:51,203 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40087 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0252adfc27b54a76a05585f3f45bf90, server=bd6592c37bb2,40087,1732486423713 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-24T22:13:51,204 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40087 {}] ipc.CallRunner(138): callId: 43 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36518 deadline: 1732486491190, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0252adfc27b54a76a05585f3f45bf90, server=bd6592c37bb2,40087,1732486423713 2024-11-24T22:13:51,204 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732486426439.f0252adfc27b54a76a05585f3f45bf90., hostname=bd6592c37bb2,40087,1732486423713, seqNum=2 , the old value is region=TestAcidGuarantees,,1732486426439.f0252adfc27b54a76a05585f3f45bf90., hostname=bd6592c37bb2,40087,1732486423713, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0252adfc27b54a76a05585f3f45bf90, server=bd6592c37bb2,40087,1732486423713 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T22:13:51,204 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732486426439.f0252adfc27b54a76a05585f3f45bf90., hostname=bd6592c37bb2,40087,1732486423713, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0252adfc27b54a76a05585f3f45bf90, server=bd6592c37bb2,40087,1732486423713 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T22:13:51,205 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732486426439.f0252adfc27b54a76a05585f3f45bf90., hostname=bd6592c37bb2,40087,1732486423713, seqNum=2 because the exception is null or not the one we care about 2024-11-24T22:13:51,205 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40087 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0252adfc27b54a76a05585f3f45bf90, server=bd6592c37bb2,40087,1732486423713 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-24T22:13:51,205 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40087 {}] ipc.CallRunner(138): callId: 15 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36548 deadline: 1732486491193, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0252adfc27b54a76a05585f3f45bf90, server=bd6592c37bb2,40087,1732486423713 2024-11-24T22:13:51,207 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732486426439.f0252adfc27b54a76a05585f3f45bf90., hostname=bd6592c37bb2,40087,1732486423713, seqNum=2 , the old value is region=TestAcidGuarantees,,1732486426439.f0252adfc27b54a76a05585f3f45bf90., hostname=bd6592c37bb2,40087,1732486423713, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0252adfc27b54a76a05585f3f45bf90, server=bd6592c37bb2,40087,1732486423713 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T22:13:51,207 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732486426439.f0252adfc27b54a76a05585f3f45bf90., hostname=bd6592c37bb2,40087,1732486423713, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0252adfc27b54a76a05585f3f45bf90, server=bd6592c37bb2,40087,1732486423713 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T22:13:51,207 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732486426439.f0252adfc27b54a76a05585f3f45bf90., hostname=bd6592c37bb2,40087,1732486423713, seqNum=2 because the exception is null or not the one we care about 2024-11-24T22:13:51,218 INFO [RS:0;bd6592c37bb2:40087-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in f0252adfc27b54a76a05585f3f45bf90/C of f0252adfc27b54a76a05585f3f45bf90 into 88c916dc5d6b43af9776f182ebf2959c(size=11.8 K), total size for store is 11.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-24T22:13:51,218 DEBUG [RS:0;bd6592c37bb2:40087-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for f0252adfc27b54a76a05585f3f45bf90: 2024-11-24T22:13:51,218 INFO [RS:0;bd6592c37bb2:40087-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732486426439.f0252adfc27b54a76a05585f3f45bf90., storeName=f0252adfc27b54a76a05585f3f45bf90/C, priority=13, startTime=1732486430962; duration=0sec 2024-11-24T22:13:51,219 DEBUG [RS:0;bd6592c37bb2:40087-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-24T22:13:51,219 DEBUG [RS:0;bd6592c37bb2:40087-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: f0252adfc27b54a76a05585f3f45bf90:C 2024-11-24T22:13:51,303 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40087 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0252adfc27b54a76a05585f3f45bf90, server=bd6592c37bb2,40087,1732486423713 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-24T22:13:51,308 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40087 {}] ipc.CallRunner(138): callId: 47 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36562 deadline: 1732486491302, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0252adfc27b54a76a05585f3f45bf90, server=bd6592c37bb2,40087,1732486423713 2024-11-24T22:13:51,309 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732486426439.f0252adfc27b54a76a05585f3f45bf90., hostname=bd6592c37bb2,40087,1732486423713, seqNum=2 , the old value is region=TestAcidGuarantees,,1732486426439.f0252adfc27b54a76a05585f3f45bf90., hostname=bd6592c37bb2,40087,1732486423713, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0252adfc27b54a76a05585f3f45bf90, server=bd6592c37bb2,40087,1732486423713 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T22:13:51,309 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40087 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0252adfc27b54a76a05585f3f45bf90, server=bd6592c37bb2,40087,1732486423713 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-24T22:13:51,309 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732486426439.f0252adfc27b54a76a05585f3f45bf90., hostname=bd6592c37bb2,40087,1732486423713, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0252adfc27b54a76a05585f3f45bf90, server=bd6592c37bb2,40087,1732486423713 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T22:13:51,309 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732486426439.f0252adfc27b54a76a05585f3f45bf90., hostname=bd6592c37bb2,40087,1732486423713, seqNum=2 because the exception is null or not the one we care about 2024-11-24T22:13:51,309 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40087 {}] ipc.CallRunner(138): callId: 27 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36590 deadline: 1732486491303, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0252adfc27b54a76a05585f3f45bf90, server=bd6592c37bb2,40087,1732486423713 2024-11-24T22:13:51,310 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732486426439.f0252adfc27b54a76a05585f3f45bf90., hostname=bd6592c37bb2,40087,1732486423713, seqNum=2 , the old value is region=TestAcidGuarantees,,1732486426439.f0252adfc27b54a76a05585f3f45bf90., hostname=bd6592c37bb2,40087,1732486423713, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0252adfc27b54a76a05585f3f45bf90, server=bd6592c37bb2,40087,1732486423713 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T22:13:51,310 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40087 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0252adfc27b54a76a05585f3f45bf90, server=bd6592c37bb2,40087,1732486423713 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-24T22:13:51,310 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732486426439.f0252adfc27b54a76a05585f3f45bf90., hostname=bd6592c37bb2,40087,1732486423713, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0252adfc27b54a76a05585f3f45bf90, server=bd6592c37bb2,40087,1732486423713 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T22:13:51,310 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732486426439.f0252adfc27b54a76a05585f3f45bf90., hostname=bd6592c37bb2,40087,1732486423713, seqNum=2 because the exception is null or not the one we care about 2024-11-24T22:13:51,310 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40087 {}] ipc.CallRunner(138): callId: 43 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36532 deadline: 1732486491303, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0252adfc27b54a76a05585f3f45bf90, server=bd6592c37bb2,40087,1732486423713 2024-11-24T22:13:51,312 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732486426439.f0252adfc27b54a76a05585f3f45bf90., hostname=bd6592c37bb2,40087,1732486423713, seqNum=2 , the old value is region=TestAcidGuarantees,,1732486426439.f0252adfc27b54a76a05585f3f45bf90., hostname=bd6592c37bb2,40087,1732486423713, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0252adfc27b54a76a05585f3f45bf90, server=bd6592c37bb2,40087,1732486423713 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T22:13:51,312 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732486426439.f0252adfc27b54a76a05585f3f45bf90., hostname=bd6592c37bb2,40087,1732486423713, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0252adfc27b54a76a05585f3f45bf90, server=bd6592c37bb2,40087,1732486423713 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T22:13:51,312 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732486426439.f0252adfc27b54a76a05585f3f45bf90., hostname=bd6592c37bb2,40087,1732486423713, seqNum=2 because the exception is null or not the one we care about 2024-11-24T22:13:51,314 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40087 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0252adfc27b54a76a05585f3f45bf90, server=bd6592c37bb2,40087,1732486423713 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-24T22:13:51,314 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40087 {}] ipc.CallRunner(138): callId: 44 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36518 deadline: 1732486491311, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0252adfc27b54a76a05585f3f45bf90, server=bd6592c37bb2,40087,1732486423713 2024-11-24T22:13:51,314 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732486426439.f0252adfc27b54a76a05585f3f45bf90., hostname=bd6592c37bb2,40087,1732486423713, seqNum=2 , the old value is region=TestAcidGuarantees,,1732486426439.f0252adfc27b54a76a05585f3f45bf90., hostname=bd6592c37bb2,40087,1732486423713, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0252adfc27b54a76a05585f3f45bf90, server=bd6592c37bb2,40087,1732486423713 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T22:13:51,315 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40087 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0252adfc27b54a76a05585f3f45bf90, server=bd6592c37bb2,40087,1732486423713 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-24T22:13:51,315 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732486426439.f0252adfc27b54a76a05585f3f45bf90., hostname=bd6592c37bb2,40087,1732486423713, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0252adfc27b54a76a05585f3f45bf90, server=bd6592c37bb2,40087,1732486423713 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T22:13:51,316 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732486426439.f0252adfc27b54a76a05585f3f45bf90., hostname=bd6592c37bb2,40087,1732486423713, seqNum=2 because the exception is null or not the one we care about 2024-11-24T22:13:51,317 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40087 {}] ipc.CallRunner(138): callId: 16 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36548 deadline: 1732486491312, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0252adfc27b54a76a05585f3f45bf90, server=bd6592c37bb2,40087,1732486423713 2024-11-24T22:13:51,317 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732486426439.f0252adfc27b54a76a05585f3f45bf90., hostname=bd6592c37bb2,40087,1732486423713, seqNum=2 , the old value is region=TestAcidGuarantees,,1732486426439.f0252adfc27b54a76a05585f3f45bf90., hostname=bd6592c37bb2,40087,1732486423713, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0252adfc27b54a76a05585f3f45bf90, server=bd6592c37bb2,40087,1732486423713 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T22:13:51,317 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732486426439.f0252adfc27b54a76a05585f3f45bf90., hostname=bd6592c37bb2,40087,1732486423713, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0252adfc27b54a76a05585f3f45bf90, server=bd6592c37bb2,40087,1732486423713 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T22:13:51,317 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732486426439.f0252adfc27b54a76a05585f3f45bf90., hostname=bd6592c37bb2,40087,1732486423713, seqNum=2 because the exception is null or not the one we care about 2024-11-24T22:13:51,479 DEBUG [RS:0;bd6592c37bb2:40087-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/f0252adfc27b54a76a05585f3f45bf90/.tmp/B/6cdf2f3301bb413f9d1e2d9b0160fee3 as hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/f0252adfc27b54a76a05585f3f45bf90/B/6cdf2f3301bb413f9d1e2d9b0160fee3 2024-11-24T22:13:51,493 INFO [RS:0;bd6592c37bb2:40087-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in f0252adfc27b54a76a05585f3f45bf90/B of f0252adfc27b54a76a05585f3f45bf90 into 6cdf2f3301bb413f9d1e2d9b0160fee3(size=11.8 K), total size for store is 11.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-24T22:13:51,493 DEBUG [RS:0;bd6592c37bb2:40087-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for f0252adfc27b54a76a05585f3f45bf90: 2024-11-24T22:13:51,493 INFO [RS:0;bd6592c37bb2:40087-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732486426439.f0252adfc27b54a76a05585f3f45bf90., storeName=f0252adfc27b54a76a05585f3f45bf90/B, priority=13, startTime=1732486430962; duration=0sec 2024-11-24T22:13:51,494 DEBUG [RS:0;bd6592c37bb2:40087-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-24T22:13:51,494 DEBUG [RS:0;bd6592c37bb2:40087-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: f0252adfc27b54a76a05585f3f45bf90:B 2024-11-24T22:13:51,497 INFO [RS_FLUSH_OPERATIONS-regionserver/bd6592c37bb2:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=24.60 KB at sequenceid=134 (bloomFilter=true), to=hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/f0252adfc27b54a76a05585f3f45bf90/.tmp/A/705ad699596346dba6cf6781c5011618 2024-11-24T22:13:51,516 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bd6592c37bb2:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/f0252adfc27b54a76a05585f3f45bf90/.tmp/B/f77f1f3e41f74085b82344614b253a1f is 50, key is test_row_0/B:col10/1732486429971/Put/seqid=0 2024-11-24T22:13:51,524 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40087 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0252adfc27b54a76a05585f3f45bf90, server=bd6592c37bb2,40087,1732486423713 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-24T22:13:51,525 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40087 {}] ipc.CallRunner(138): callId: 28 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36590 deadline: 1732486491521, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0252adfc27b54a76a05585f3f45bf90, server=bd6592c37bb2,40087,1732486423713 2024-11-24T22:13:51,525 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732486426439.f0252adfc27b54a76a05585f3f45bf90., hostname=bd6592c37bb2,40087,1732486423713, seqNum=2 , the old value is region=TestAcidGuarantees,,1732486426439.f0252adfc27b54a76a05585f3f45bf90., hostname=bd6592c37bb2,40087,1732486423713, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0252adfc27b54a76a05585f3f45bf90, server=bd6592c37bb2,40087,1732486423713 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T22:13:51,525 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732486426439.f0252adfc27b54a76a05585f3f45bf90., hostname=bd6592c37bb2,40087,1732486423713, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0252adfc27b54a76a05585f3f45bf90, server=bd6592c37bb2,40087,1732486423713 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T22:13:51,525 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40087 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0252adfc27b54a76a05585f3f45bf90, server=bd6592c37bb2,40087,1732486423713 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-24T22:13:51,526 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732486426439.f0252adfc27b54a76a05585f3f45bf90., hostname=bd6592c37bb2,40087,1732486423713, seqNum=2 because the exception is null or not the one we care about 2024-11-24T22:13:51,526 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40087 {}] ipc.CallRunner(138): callId: 44 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36532 deadline: 1732486491521, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0252adfc27b54a76a05585f3f45bf90, server=bd6592c37bb2,40087,1732486423713 2024-11-24T22:13:51,526 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732486426439.f0252adfc27b54a76a05585f3f45bf90., hostname=bd6592c37bb2,40087,1732486423713, seqNum=2 , the old value is region=TestAcidGuarantees,,1732486426439.f0252adfc27b54a76a05585f3f45bf90., hostname=bd6592c37bb2,40087,1732486423713, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0252adfc27b54a76a05585f3f45bf90, server=bd6592c37bb2,40087,1732486423713 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T22:13:51,526 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732486426439.f0252adfc27b54a76a05585f3f45bf90., hostname=bd6592c37bb2,40087,1732486423713, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0252adfc27b54a76a05585f3f45bf90, server=bd6592c37bb2,40087,1732486423713 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T22:13:51,527 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732486426439.f0252adfc27b54a76a05585f3f45bf90., hostname=bd6592c37bb2,40087,1732486423713, seqNum=2 because the exception is null or not the one we care about 2024-11-24T22:13:51,527 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40087 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0252adfc27b54a76a05585f3f45bf90, server=bd6592c37bb2,40087,1732486423713 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-24T22:13:51,527 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40087 {}] ipc.CallRunner(138): callId: 45 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36518 deadline: 1732486491522, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0252adfc27b54a76a05585f3f45bf90, server=bd6592c37bb2,40087,1732486423713 2024-11-24T22:13:51,527 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732486426439.f0252adfc27b54a76a05585f3f45bf90., hostname=bd6592c37bb2,40087,1732486423713, seqNum=2 , the old value is region=TestAcidGuarantees,,1732486426439.f0252adfc27b54a76a05585f3f45bf90., hostname=bd6592c37bb2,40087,1732486423713, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0252adfc27b54a76a05585f3f45bf90, server=bd6592c37bb2,40087,1732486423713 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T22:13:51,527 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732486426439.f0252adfc27b54a76a05585f3f45bf90., hostname=bd6592c37bb2,40087,1732486423713, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0252adfc27b54a76a05585f3f45bf90, server=bd6592c37bb2,40087,1732486423713 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T22:13:51,527 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732486426439.f0252adfc27b54a76a05585f3f45bf90., hostname=bd6592c37bb2,40087,1732486423713, seqNum=2 because the exception is null or not the one we care about 2024-11-24T22:13:51,527 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40087 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0252adfc27b54a76a05585f3f45bf90, server=bd6592c37bb2,40087,1732486423713 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-24T22:13:51,528 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40087 {}] ipc.CallRunner(138): callId: 48 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36562 deadline: 1732486491522, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0252adfc27b54a76a05585f3f45bf90, server=bd6592c37bb2,40087,1732486423713 2024-11-24T22:13:51,528 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732486426439.f0252adfc27b54a76a05585f3f45bf90., hostname=bd6592c37bb2,40087,1732486423713, seqNum=2 , the old value is region=TestAcidGuarantees,,1732486426439.f0252adfc27b54a76a05585f3f45bf90., hostname=bd6592c37bb2,40087,1732486423713, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0252adfc27b54a76a05585f3f45bf90, server=bd6592c37bb2,40087,1732486423713 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T22:13:51,528 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732486426439.f0252adfc27b54a76a05585f3f45bf90., hostname=bd6592c37bb2,40087,1732486423713, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0252adfc27b54a76a05585f3f45bf90, server=bd6592c37bb2,40087,1732486423713 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T22:13:51,528 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40087 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0252adfc27b54a76a05585f3f45bf90, server=bd6592c37bb2,40087,1732486423713 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-24T22:13:51,528 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732486426439.f0252adfc27b54a76a05585f3f45bf90., hostname=bd6592c37bb2,40087,1732486423713, seqNum=2 because the exception is null or not the one we care about 2024-11-24T22:13:51,529 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40087 {}] ipc.CallRunner(138): callId: 17 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36548 deadline: 1732486491522, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0252adfc27b54a76a05585f3f45bf90, server=bd6592c37bb2,40087,1732486423713 2024-11-24T22:13:51,529 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732486426439.f0252adfc27b54a76a05585f3f45bf90., hostname=bd6592c37bb2,40087,1732486423713, seqNum=2 , the old value is region=TestAcidGuarantees,,1732486426439.f0252adfc27b54a76a05585f3f45bf90., hostname=bd6592c37bb2,40087,1732486423713, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0252adfc27b54a76a05585f3f45bf90, server=bd6592c37bb2,40087,1732486423713 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T22:13:51,529 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732486426439.f0252adfc27b54a76a05585f3f45bf90., hostname=bd6592c37bb2,40087,1732486423713, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0252adfc27b54a76a05585f3f45bf90, server=bd6592c37bb2,40087,1732486423713 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T22:13:51,529 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732486426439.f0252adfc27b54a76a05585f3f45bf90., hostname=bd6592c37bb2,40087,1732486423713, seqNum=2 because the exception is null or not the one we care about 2024-11-24T22:13:51,542 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45929 is added to blk_1073741850_1026 (size=12101) 2024-11-24T22:13:51,546 INFO [RS_FLUSH_OPERATIONS-regionserver/bd6592c37bb2:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=24.60 KB at sequenceid=134 (bloomFilter=true), to=hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/f0252adfc27b54a76a05585f3f45bf90/.tmp/B/f77f1f3e41f74085b82344614b253a1f 2024-11-24T22:13:51,566 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bd6592c37bb2:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/f0252adfc27b54a76a05585f3f45bf90/.tmp/C/7fadf24fe8a7459ea252bf9e20a561a7 is 50, key is test_row_0/C:col10/1732486429971/Put/seqid=0 2024-11-24T22:13:51,572 WARN [HBase-Metrics2-1 {}] impl.MetricsConfig(138): Cannot locate configuration: tried hadoop-metrics2-hbase.properties,hadoop-metrics2.properties 2024-11-24T22:13:51,615 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45929 is added to blk_1073741851_1027 (size=12101) 2024-11-24T22:13:51,670 DEBUG [HBase-Metrics2-1 {}] regionserver.MetricsTableSourceImpl(133): Creating new MetricsTableSourceImpl for table 'hbase:meta' 2024-11-24T22:13:51,832 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40087 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0252adfc27b54a76a05585f3f45bf90, server=bd6592c37bb2,40087,1732486423713 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-24T22:13:51,833 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40087 {}] ipc.CallRunner(138): callId: 29 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36590 deadline: 1732486491831, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0252adfc27b54a76a05585f3f45bf90, server=bd6592c37bb2,40087,1732486423713 2024-11-24T22:13:51,833 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40087 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0252adfc27b54a76a05585f3f45bf90, server=bd6592c37bb2,40087,1732486423713 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-24T22:13:51,833 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732486426439.f0252adfc27b54a76a05585f3f45bf90., hostname=bd6592c37bb2,40087,1732486423713, seqNum=2 , the old value is region=TestAcidGuarantees,,1732486426439.f0252adfc27b54a76a05585f3f45bf90., hostname=bd6592c37bb2,40087,1732486423713, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0252adfc27b54a76a05585f3f45bf90, server=bd6592c37bb2,40087,1732486423713 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T22:13:51,834 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40087 {}] ipc.CallRunner(138): callId: 45 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36532 deadline: 1732486491831, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0252adfc27b54a76a05585f3f45bf90, server=bd6592c37bb2,40087,1732486423713 2024-11-24T22:13:51,834 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732486426439.f0252adfc27b54a76a05585f3f45bf90., hostname=bd6592c37bb2,40087,1732486423713, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0252adfc27b54a76a05585f3f45bf90, server=bd6592c37bb2,40087,1732486423713 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T22:13:51,834 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732486426439.f0252adfc27b54a76a05585f3f45bf90., hostname=bd6592c37bb2,40087,1732486423713, seqNum=2 because the exception is null or not the one we care about 2024-11-24T22:13:51,834 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732486426439.f0252adfc27b54a76a05585f3f45bf90., hostname=bd6592c37bb2,40087,1732486423713, seqNum=2 , the old value is region=TestAcidGuarantees,,1732486426439.f0252adfc27b54a76a05585f3f45bf90., hostname=bd6592c37bb2,40087,1732486423713, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0252adfc27b54a76a05585f3f45bf90, server=bd6592c37bb2,40087,1732486423713 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T22:13:51,834 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732486426439.f0252adfc27b54a76a05585f3f45bf90., hostname=bd6592c37bb2,40087,1732486423713, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0252adfc27b54a76a05585f3f45bf90, server=bd6592c37bb2,40087,1732486423713 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T22:13:51,834 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732486426439.f0252adfc27b54a76a05585f3f45bf90., hostname=bd6592c37bb2,40087,1732486423713, seqNum=2 because the exception is null or not the one we care about 2024-11-24T22:13:51,848 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40087 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0252adfc27b54a76a05585f3f45bf90, server=bd6592c37bb2,40087,1732486423713 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-24T22:13:51,848 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40087 {}] ipc.CallRunner(138): callId: 49 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36562 deadline: 1732486491842, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0252adfc27b54a76a05585f3f45bf90, server=bd6592c37bb2,40087,1732486423713 2024-11-24T22:13:51,849 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732486426439.f0252adfc27b54a76a05585f3f45bf90., hostname=bd6592c37bb2,40087,1732486423713, seqNum=2 , the old value is region=TestAcidGuarantees,,1732486426439.f0252adfc27b54a76a05585f3f45bf90., hostname=bd6592c37bb2,40087,1732486423713, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0252adfc27b54a76a05585f3f45bf90, server=bd6592c37bb2,40087,1732486423713 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T22:13:51,849 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732486426439.f0252adfc27b54a76a05585f3f45bf90., hostname=bd6592c37bb2,40087,1732486423713, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0252adfc27b54a76a05585f3f45bf90, server=bd6592c37bb2,40087,1732486423713 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T22:13:51,849 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40087 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0252adfc27b54a76a05585f3f45bf90, server=bd6592c37bb2,40087,1732486423713 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-24T22:13:51,849 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732486426439.f0252adfc27b54a76a05585f3f45bf90., hostname=bd6592c37bb2,40087,1732486423713, seqNum=2 because the exception is null or not the one we care about 2024-11-24T22:13:51,849 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40087 {}] ipc.CallRunner(138): callId: 46 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36518 deadline: 1732486491842, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0252adfc27b54a76a05585f3f45bf90, server=bd6592c37bb2,40087,1732486423713 2024-11-24T22:13:51,849 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40087 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0252adfc27b54a76a05585f3f45bf90, server=bd6592c37bb2,40087,1732486423713 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-24T22:13:51,850 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40087 {}] ipc.CallRunner(138): callId: 18 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36548 deadline: 1732486491843, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0252adfc27b54a76a05585f3f45bf90, server=bd6592c37bb2,40087,1732486423713 2024-11-24T22:13:51,850 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732486426439.f0252adfc27b54a76a05585f3f45bf90., hostname=bd6592c37bb2,40087,1732486423713, seqNum=2 , the old value is region=TestAcidGuarantees,,1732486426439.f0252adfc27b54a76a05585f3f45bf90., hostname=bd6592c37bb2,40087,1732486423713, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0252adfc27b54a76a05585f3f45bf90, server=bd6592c37bb2,40087,1732486423713 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T22:13:51,850 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732486426439.f0252adfc27b54a76a05585f3f45bf90., hostname=bd6592c37bb2,40087,1732486423713, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0252adfc27b54a76a05585f3f45bf90, server=bd6592c37bb2,40087,1732486423713 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T22:13:51,850 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732486426439.f0252adfc27b54a76a05585f3f45bf90., hostname=bd6592c37bb2,40087,1732486423713, seqNum=2 because the exception is null or not the one we care about 2024-11-24T22:13:51,850 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732486426439.f0252adfc27b54a76a05585f3f45bf90., hostname=bd6592c37bb2,40087,1732486423713, seqNum=2 , the old value is region=TestAcidGuarantees,,1732486426439.f0252adfc27b54a76a05585f3f45bf90., hostname=bd6592c37bb2,40087,1732486423713, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0252adfc27b54a76a05585f3f45bf90, server=bd6592c37bb2,40087,1732486423713 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T22:13:51,851 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732486426439.f0252adfc27b54a76a05585f3f45bf90., hostname=bd6592c37bb2,40087,1732486423713, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0252adfc27b54a76a05585f3f45bf90, server=bd6592c37bb2,40087,1732486423713 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T22:13:51,851 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732486426439.f0252adfc27b54a76a05585f3f45bf90., hostname=bd6592c37bb2,40087,1732486423713, seqNum=2 because the exception is null or not the one we care about 2024-11-24T22:13:52,017 INFO [RS_FLUSH_OPERATIONS-regionserver/bd6592c37bb2:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=24.60 KB at sequenceid=134 (bloomFilter=true), to=hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/f0252adfc27b54a76a05585f3f45bf90/.tmp/C/7fadf24fe8a7459ea252bf9e20a561a7 2024-11-24T22:13:52,030 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bd6592c37bb2:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/f0252adfc27b54a76a05585f3f45bf90/.tmp/A/705ad699596346dba6cf6781c5011618 as hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/f0252adfc27b54a76a05585f3f45bf90/A/705ad699596346dba6cf6781c5011618 2024-11-24T22:13:52,042 INFO [RS_FLUSH_OPERATIONS-regionserver/bd6592c37bb2:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/f0252adfc27b54a76a05585f3f45bf90/A/705ad699596346dba6cf6781c5011618, entries=150, sequenceid=134, filesize=11.8 K 2024-11-24T22:13:52,044 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bd6592c37bb2:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/f0252adfc27b54a76a05585f3f45bf90/.tmp/B/f77f1f3e41f74085b82344614b253a1f as hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/f0252adfc27b54a76a05585f3f45bf90/B/f77f1f3e41f74085b82344614b253a1f 2024-11-24T22:13:52,055 INFO [RS_FLUSH_OPERATIONS-regionserver/bd6592c37bb2:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/f0252adfc27b54a76a05585f3f45bf90/B/f77f1f3e41f74085b82344614b253a1f, entries=150, sequenceid=134, filesize=11.8 K 2024-11-24T22:13:52,059 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bd6592c37bb2:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/f0252adfc27b54a76a05585f3f45bf90/.tmp/C/7fadf24fe8a7459ea252bf9e20a561a7 as hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/f0252adfc27b54a76a05585f3f45bf90/C/7fadf24fe8a7459ea252bf9e20a561a7 2024-11-24T22:13:52,074 INFO [RS_FLUSH_OPERATIONS-regionserver/bd6592c37bb2:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/f0252adfc27b54a76a05585f3f45bf90/C/7fadf24fe8a7459ea252bf9e20a561a7, entries=150, sequenceid=134, filesize=11.8 K 2024-11-24T22:13:52,076 INFO [RS_FLUSH_OPERATIONS-regionserver/bd6592c37bb2:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.HRegion(3140): Finished flush of dataSize ~73.80 KB/75570, heapSize ~219.84 KB/225120, currentSize=100.63 KB/103050 for f0252adfc27b54a76a05585f3f45bf90 in 1060ms, sequenceid=134, compaction requested=false 2024-11-24T22:13:52,076 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bd6592c37bb2:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.HRegion(2603): Flush status journal for f0252adfc27b54a76a05585f3f45bf90: 2024-11-24T22:13:52,076 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bd6592c37bb2:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732486426439.f0252adfc27b54a76a05585f3f45bf90. 2024-11-24T22:13:52,077 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bd6592c37bb2:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=8 2024-11-24T22:13:52,077 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37701 {}] master.HMaster(4169): Remote procedure done, pid=8 2024-11-24T22:13:52,084 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=8, resume processing ppid=7 2024-11-24T22:13:52,084 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=8, ppid=7, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 3.3550 sec 2024-11-24T22:13:52,093 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=7, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=7, table=TestAcidGuarantees in 3.3750 sec 2024-11-24T22:13:52,344 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40087 {}] regionserver.HRegion(8855): Flush requested on f0252adfc27b54a76a05585f3f45bf90 2024-11-24T22:13:52,344 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing f0252adfc27b54a76a05585f3f45bf90 3/3 column families, dataSize=107.34 KB heapSize=319.50 KB 2024-11-24T22:13:52,346 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK f0252adfc27b54a76a05585f3f45bf90, store=A 2024-11-24T22:13:52,346 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-24T22:13:52,346 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK f0252adfc27b54a76a05585f3f45bf90, store=B 2024-11-24T22:13:52,346 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-24T22:13:52,346 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK f0252adfc27b54a76a05585f3f45bf90, store=C 2024-11-24T22:13:52,346 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-24T22:13:52,362 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/f0252adfc27b54a76a05585f3f45bf90/.tmp/A/ab0287723e304b5ba215650833ca6579 is 50, key is test_row_0/A:col10/1732486432344/Put/seqid=0 2024-11-24T22:13:52,379 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45929 is added to blk_1073741852_1028 (size=12151) 2024-11-24T22:13:52,386 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40087 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0252adfc27b54a76a05585f3f45bf90, server=bd6592c37bb2,40087,1732486423713 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-24T22:13:52,387 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40087 {}] ipc.CallRunner(138): callId: 48 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36518 deadline: 1732486492370, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0252adfc27b54a76a05585f3f45bf90, server=bd6592c37bb2,40087,1732486423713 2024-11-24T22:13:52,387 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732486426439.f0252adfc27b54a76a05585f3f45bf90., hostname=bd6592c37bb2,40087,1732486423713, seqNum=2 , the old value is region=TestAcidGuarantees,,1732486426439.f0252adfc27b54a76a05585f3f45bf90., hostname=bd6592c37bb2,40087,1732486423713, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0252adfc27b54a76a05585f3f45bf90, server=bd6592c37bb2,40087,1732486423713 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T22:13:52,387 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40087 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0252adfc27b54a76a05585f3f45bf90, server=bd6592c37bb2,40087,1732486423713 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-24T22:13:52,387 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732486426439.f0252adfc27b54a76a05585f3f45bf90., hostname=bd6592c37bb2,40087,1732486423713, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0252adfc27b54a76a05585f3f45bf90, server=bd6592c37bb2,40087,1732486423713 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T22:13:52,388 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732486426439.f0252adfc27b54a76a05585f3f45bf90., hostname=bd6592c37bb2,40087,1732486423713, seqNum=2 because the exception is null or not the one we care about 2024-11-24T22:13:52,388 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40087 {}] ipc.CallRunner(138): callId: 51 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36562 deadline: 1732486492371, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0252adfc27b54a76a05585f3f45bf90, server=bd6592c37bb2,40087,1732486423713 2024-11-24T22:13:52,387 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40087 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0252adfc27b54a76a05585f3f45bf90, server=bd6592c37bb2,40087,1732486423713 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-24T22:13:52,388 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40087 {}] ipc.CallRunner(138): callId: 20 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36548 deadline: 1732486492372, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0252adfc27b54a76a05585f3f45bf90, server=bd6592c37bb2,40087,1732486423713 2024-11-24T22:13:52,388 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40087 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0252adfc27b54a76a05585f3f45bf90, server=bd6592c37bb2,40087,1732486423713 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-24T22:13:52,389 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40087 {}] ipc.CallRunner(138): callId: 34 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36590 deadline: 1732486492373, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0252adfc27b54a76a05585f3f45bf90, server=bd6592c37bb2,40087,1732486423713 2024-11-24T22:13:52,389 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40087 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0252adfc27b54a76a05585f3f45bf90, server=bd6592c37bb2,40087,1732486423713 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-24T22:13:52,389 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40087 {}] ipc.CallRunner(138): callId: 50 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36532 deadline: 1732486492375, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0252adfc27b54a76a05585f3f45bf90, server=bd6592c37bb2,40087,1732486423713 2024-11-24T22:13:52,390 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732486426439.f0252adfc27b54a76a05585f3f45bf90., hostname=bd6592c37bb2,40087,1732486423713, seqNum=2 , the old value is region=TestAcidGuarantees,,1732486426439.f0252adfc27b54a76a05585f3f45bf90., hostname=bd6592c37bb2,40087,1732486423713, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0252adfc27b54a76a05585f3f45bf90, server=bd6592c37bb2,40087,1732486423713 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T22:13:52,390 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732486426439.f0252adfc27b54a76a05585f3f45bf90., hostname=bd6592c37bb2,40087,1732486423713, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0252adfc27b54a76a05585f3f45bf90, server=bd6592c37bb2,40087,1732486423713 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T22:13:52,390 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732486426439.f0252adfc27b54a76a05585f3f45bf90., hostname=bd6592c37bb2,40087,1732486423713, seqNum=2 because the exception is null or not the one we care about 2024-11-24T22:13:52,391 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732486426439.f0252adfc27b54a76a05585f3f45bf90., hostname=bd6592c37bb2,40087,1732486423713, seqNum=2 , the old value is region=TestAcidGuarantees,,1732486426439.f0252adfc27b54a76a05585f3f45bf90., hostname=bd6592c37bb2,40087,1732486423713, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0252adfc27b54a76a05585f3f45bf90, server=bd6592c37bb2,40087,1732486423713 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T22:13:52,391 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732486426439.f0252adfc27b54a76a05585f3f45bf90., hostname=bd6592c37bb2,40087,1732486423713, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0252adfc27b54a76a05585f3f45bf90, server=bd6592c37bb2,40087,1732486423713 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T22:13:52,391 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732486426439.f0252adfc27b54a76a05585f3f45bf90., hostname=bd6592c37bb2,40087,1732486423713, seqNum=2 because the exception is null or not the one we care about 2024-11-24T22:13:52,391 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732486426439.f0252adfc27b54a76a05585f3f45bf90., hostname=bd6592c37bb2,40087,1732486423713, seqNum=2 , the old value is region=TestAcidGuarantees,,1732486426439.f0252adfc27b54a76a05585f3f45bf90., hostname=bd6592c37bb2,40087,1732486423713, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0252adfc27b54a76a05585f3f45bf90, server=bd6592c37bb2,40087,1732486423713 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T22:13:52,391 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732486426439.f0252adfc27b54a76a05585f3f45bf90., hostname=bd6592c37bb2,40087,1732486423713, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0252adfc27b54a76a05585f3f45bf90, server=bd6592c37bb2,40087,1732486423713 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T22:13:52,392 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732486426439.f0252adfc27b54a76a05585f3f45bf90., hostname=bd6592c37bb2,40087,1732486423713, seqNum=2 because the exception is null or not the one we care about 2024-11-24T22:13:52,392 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732486426439.f0252adfc27b54a76a05585f3f45bf90., hostname=bd6592c37bb2,40087,1732486423713, seqNum=2 , the old value is region=TestAcidGuarantees,,1732486426439.f0252adfc27b54a76a05585f3f45bf90., hostname=bd6592c37bb2,40087,1732486423713, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0252adfc27b54a76a05585f3f45bf90, server=bd6592c37bb2,40087,1732486423713 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T22:13:52,392 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732486426439.f0252adfc27b54a76a05585f3f45bf90., hostname=bd6592c37bb2,40087,1732486423713, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0252adfc27b54a76a05585f3f45bf90, server=bd6592c37bb2,40087,1732486423713 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T22:13:52,392 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732486426439.f0252adfc27b54a76a05585f3f45bf90., hostname=bd6592c37bb2,40087,1732486423713, seqNum=2 because the exception is null or not the one we care about 2024-11-24T22:13:52,493 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40087 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0252adfc27b54a76a05585f3f45bf90, server=bd6592c37bb2,40087,1732486423713 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-24T22:13:52,494 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40087 {}] ipc.CallRunner(138): callId: 49 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36518 deadline: 1732486492491, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0252adfc27b54a76a05585f3f45bf90, server=bd6592c37bb2,40087,1732486423713 2024-11-24T22:13:52,494 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732486426439.f0252adfc27b54a76a05585f3f45bf90., hostname=bd6592c37bb2,40087,1732486423713, seqNum=2 , the old value is region=TestAcidGuarantees,,1732486426439.f0252adfc27b54a76a05585f3f45bf90., hostname=bd6592c37bb2,40087,1732486423713, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0252adfc27b54a76a05585f3f45bf90, server=bd6592c37bb2,40087,1732486423713 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T22:13:52,494 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732486426439.f0252adfc27b54a76a05585f3f45bf90., hostname=bd6592c37bb2,40087,1732486423713, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0252adfc27b54a76a05585f3f45bf90, server=bd6592c37bb2,40087,1732486423713 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T22:13:52,494 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732486426439.f0252adfc27b54a76a05585f3f45bf90., hostname=bd6592c37bb2,40087,1732486423713, seqNum=2 because the exception is null or not the one we care about 2024-11-24T22:13:52,510 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40087 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0252adfc27b54a76a05585f3f45bf90, server=bd6592c37bb2,40087,1732486423713 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-24T22:13:52,511 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40087 {}] ipc.CallRunner(138): callId: 21 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36548 deadline: 1732486492502, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0252adfc27b54a76a05585f3f45bf90, server=bd6592c37bb2,40087,1732486423713 2024-11-24T22:13:52,512 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732486426439.f0252adfc27b54a76a05585f3f45bf90., hostname=bd6592c37bb2,40087,1732486423713, seqNum=2 , the old value is region=TestAcidGuarantees,,1732486426439.f0252adfc27b54a76a05585f3f45bf90., hostname=bd6592c37bb2,40087,1732486423713, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0252adfc27b54a76a05585f3f45bf90, server=bd6592c37bb2,40087,1732486423713 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T22:13:52,511 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40087 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0252adfc27b54a76a05585f3f45bf90, server=bd6592c37bb2,40087,1732486423713 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-24T22:13:52,512 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732486426439.f0252adfc27b54a76a05585f3f45bf90., hostname=bd6592c37bb2,40087,1732486423713, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0252adfc27b54a76a05585f3f45bf90, server=bd6592c37bb2,40087,1732486423713 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T22:13:52,512 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732486426439.f0252adfc27b54a76a05585f3f45bf90., hostname=bd6592c37bb2,40087,1732486423713, seqNum=2 because the exception is null or not the one we care about 2024-11-24T22:13:52,512 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40087 {}] ipc.CallRunner(138): callId: 52 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36562 deadline: 1732486492502, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0252adfc27b54a76a05585f3f45bf90, server=bd6592c37bb2,40087,1732486423713 2024-11-24T22:13:52,513 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732486426439.f0252adfc27b54a76a05585f3f45bf90., hostname=bd6592c37bb2,40087,1732486423713, seqNum=2 , the old value is region=TestAcidGuarantees,,1732486426439.f0252adfc27b54a76a05585f3f45bf90., hostname=bd6592c37bb2,40087,1732486423713, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0252adfc27b54a76a05585f3f45bf90, server=bd6592c37bb2,40087,1732486423713 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T22:13:52,513 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40087 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0252adfc27b54a76a05585f3f45bf90, server=bd6592c37bb2,40087,1732486423713 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-24T22:13:52,513 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732486426439.f0252adfc27b54a76a05585f3f45bf90., hostname=bd6592c37bb2,40087,1732486423713, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0252adfc27b54a76a05585f3f45bf90, server=bd6592c37bb2,40087,1732486423713 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T22:13:52,513 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732486426439.f0252adfc27b54a76a05585f3f45bf90., hostname=bd6592c37bb2,40087,1732486423713, seqNum=2 because the exception is null or not the one we care about 2024-11-24T22:13:52,513 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40087 {}] ipc.CallRunner(138): callId: 35 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36590 deadline: 1732486492502, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0252adfc27b54a76a05585f3f45bf90, server=bd6592c37bb2,40087,1732486423713 2024-11-24T22:13:52,513 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40087 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0252adfc27b54a76a05585f3f45bf90, server=bd6592c37bb2,40087,1732486423713 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-24T22:13:52,514 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40087 {}] ipc.CallRunner(138): callId: 51 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36532 deadline: 1732486492503, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0252adfc27b54a76a05585f3f45bf90, server=bd6592c37bb2,40087,1732486423713 2024-11-24T22:13:52,514 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732486426439.f0252adfc27b54a76a05585f3f45bf90., hostname=bd6592c37bb2,40087,1732486423713, seqNum=2 , the old value is region=TestAcidGuarantees,,1732486426439.f0252adfc27b54a76a05585f3f45bf90., hostname=bd6592c37bb2,40087,1732486423713, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0252adfc27b54a76a05585f3f45bf90, server=bd6592c37bb2,40087,1732486423713 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T22:13:52,514 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732486426439.f0252adfc27b54a76a05585f3f45bf90., hostname=bd6592c37bb2,40087,1732486423713, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0252adfc27b54a76a05585f3f45bf90, server=bd6592c37bb2,40087,1732486423713 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T22:13:52,514 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732486426439.f0252adfc27b54a76a05585f3f45bf90., hostname=bd6592c37bb2,40087,1732486423713, seqNum=2 because the exception is null or not the one we care about 2024-11-24T22:13:52,514 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732486426439.f0252adfc27b54a76a05585f3f45bf90., hostname=bd6592c37bb2,40087,1732486423713, seqNum=2 , the old value is region=TestAcidGuarantees,,1732486426439.f0252adfc27b54a76a05585f3f45bf90., hostname=bd6592c37bb2,40087,1732486423713, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0252adfc27b54a76a05585f3f45bf90, server=bd6592c37bb2,40087,1732486423713 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T22:13:52,514 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732486426439.f0252adfc27b54a76a05585f3f45bf90., hostname=bd6592c37bb2,40087,1732486423713, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0252adfc27b54a76a05585f3f45bf90, server=bd6592c37bb2,40087,1732486423713 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T22:13:52,514 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732486426439.f0252adfc27b54a76a05585f3f45bf90., hostname=bd6592c37bb2,40087,1732486423713, seqNum=2 because the exception is null or not the one we care about 2024-11-24T22:13:52,705 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40087 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0252adfc27b54a76a05585f3f45bf90, server=bd6592c37bb2,40087,1732486423713 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-24T22:13:52,705 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40087 {}] ipc.CallRunner(138): callId: 50 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36518 deadline: 1732486492701, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0252adfc27b54a76a05585f3f45bf90, server=bd6592c37bb2,40087,1732486423713 2024-11-24T22:13:52,706 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732486426439.f0252adfc27b54a76a05585f3f45bf90., hostname=bd6592c37bb2,40087,1732486423713, seqNum=2 , the old value is region=TestAcidGuarantees,,1732486426439.f0252adfc27b54a76a05585f3f45bf90., hostname=bd6592c37bb2,40087,1732486423713, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0252adfc27b54a76a05585f3f45bf90, server=bd6592c37bb2,40087,1732486423713 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T22:13:52,706 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732486426439.f0252adfc27b54a76a05585f3f45bf90., hostname=bd6592c37bb2,40087,1732486423713, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0252adfc27b54a76a05585f3f45bf90, server=bd6592c37bb2,40087,1732486423713 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T22:13:52,706 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732486426439.f0252adfc27b54a76a05585f3f45bf90., hostname=bd6592c37bb2,40087,1732486423713, seqNum=2 because the exception is null or not the one we care about 2024-11-24T22:13:52,726 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40087 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0252adfc27b54a76a05585f3f45bf90, server=bd6592c37bb2,40087,1732486423713 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-24T22:13:52,726 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40087 {}] ipc.CallRunner(138): callId: 53 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36562 deadline: 1732486492721, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0252adfc27b54a76a05585f3f45bf90, server=bd6592c37bb2,40087,1732486423713 2024-11-24T22:13:52,727 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40087 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0252adfc27b54a76a05585f3f45bf90, server=bd6592c37bb2,40087,1732486423713 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-24T22:13:52,727 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732486426439.f0252adfc27b54a76a05585f3f45bf90., hostname=bd6592c37bb2,40087,1732486423713, seqNum=2 , the old value is region=TestAcidGuarantees,,1732486426439.f0252adfc27b54a76a05585f3f45bf90., hostname=bd6592c37bb2,40087,1732486423713, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0252adfc27b54a76a05585f3f45bf90, server=bd6592c37bb2,40087,1732486423713 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T22:13:52,727 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40087 {}] ipc.CallRunner(138): callId: 36 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36590 deadline: 1732486492724, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0252adfc27b54a76a05585f3f45bf90, server=bd6592c37bb2,40087,1732486423713 2024-11-24T22:13:52,727 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732486426439.f0252adfc27b54a76a05585f3f45bf90., hostname=bd6592c37bb2,40087,1732486423713, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0252adfc27b54a76a05585f3f45bf90, server=bd6592c37bb2,40087,1732486423713 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T22:13:52,727 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732486426439.f0252adfc27b54a76a05585f3f45bf90., hostname=bd6592c37bb2,40087,1732486423713, seqNum=2 because the exception is null or not the one we care about 2024-11-24T22:13:52,727 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732486426439.f0252adfc27b54a76a05585f3f45bf90., hostname=bd6592c37bb2,40087,1732486423713, seqNum=2 , the old value is region=TestAcidGuarantees,,1732486426439.f0252adfc27b54a76a05585f3f45bf90., hostname=bd6592c37bb2,40087,1732486423713, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0252adfc27b54a76a05585f3f45bf90, server=bd6592c37bb2,40087,1732486423713 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T22:13:52,727 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732486426439.f0252adfc27b54a76a05585f3f45bf90., hostname=bd6592c37bb2,40087,1732486423713, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0252adfc27b54a76a05585f3f45bf90, server=bd6592c37bb2,40087,1732486423713 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T22:13:52,728 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732486426439.f0252adfc27b54a76a05585f3f45bf90., hostname=bd6592c37bb2,40087,1732486423713, seqNum=2 because the exception is null or not the one we care about 2024-11-24T22:13:52,728 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40087 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0252adfc27b54a76a05585f3f45bf90, server=bd6592c37bb2,40087,1732486423713 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-24T22:13:52,728 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40087 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0252adfc27b54a76a05585f3f45bf90, server=bd6592c37bb2,40087,1732486423713 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-24T22:13:52,728 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40087 {}] ipc.CallRunner(138): callId: 22 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36548 deadline: 1732486492724, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0252adfc27b54a76a05585f3f45bf90, server=bd6592c37bb2,40087,1732486423713 2024-11-24T22:13:52,729 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40087 {}] ipc.CallRunner(138): callId: 52 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36532 deadline: 1732486492725, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0252adfc27b54a76a05585f3f45bf90, server=bd6592c37bb2,40087,1732486423713 2024-11-24T22:13:52,729 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732486426439.f0252adfc27b54a76a05585f3f45bf90., hostname=bd6592c37bb2,40087,1732486423713, seqNum=2 , the old value is region=TestAcidGuarantees,,1732486426439.f0252adfc27b54a76a05585f3f45bf90., hostname=bd6592c37bb2,40087,1732486423713, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0252adfc27b54a76a05585f3f45bf90, server=bd6592c37bb2,40087,1732486423713 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T22:13:52,729 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732486426439.f0252adfc27b54a76a05585f3f45bf90., hostname=bd6592c37bb2,40087,1732486423713, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0252adfc27b54a76a05585f3f45bf90, server=bd6592c37bb2,40087,1732486423713 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T22:13:52,729 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732486426439.f0252adfc27b54a76a05585f3f45bf90., hostname=bd6592c37bb2,40087,1732486423713, seqNum=2 because the exception is null or not the one we care about 2024-11-24T22:13:52,732 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732486426439.f0252adfc27b54a76a05585f3f45bf90., hostname=bd6592c37bb2,40087,1732486423713, seqNum=2 , the old value is region=TestAcidGuarantees,,1732486426439.f0252adfc27b54a76a05585f3f45bf90., hostname=bd6592c37bb2,40087,1732486423713, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0252adfc27b54a76a05585f3f45bf90, server=bd6592c37bb2,40087,1732486423713 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T22:13:52,733 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732486426439.f0252adfc27b54a76a05585f3f45bf90., hostname=bd6592c37bb2,40087,1732486423713, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0252adfc27b54a76a05585f3f45bf90, server=bd6592c37bb2,40087,1732486423713 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T22:13:52,733 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732486426439.f0252adfc27b54a76a05585f3f45bf90., hostname=bd6592c37bb2,40087,1732486423713, seqNum=2 because the exception is null or not the one we care about 2024-11-24T22:13:52,781 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=38.02 KB at sequenceid=157 (bloomFilter=true), to=hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/f0252adfc27b54a76a05585f3f45bf90/.tmp/A/ab0287723e304b5ba215650833ca6579 2024-11-24T22:13:52,805 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/f0252adfc27b54a76a05585f3f45bf90/.tmp/B/dc390cb315b24517a00c5d625773456e is 50, key is test_row_0/B:col10/1732486432344/Put/seqid=0 2024-11-24T22:13:52,832 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45929 is added to blk_1073741853_1029 (size=12151) 2024-11-24T22:13:52,881 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37701 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=7 2024-11-24T22:13:52,882 INFO [RPCClient-NioEventLoopGroup-4-8 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-24T22:13:52,882 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37701 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-24T22:13:52,885 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37701 {}] procedure2.ProcedureExecutor(1139): Stored pid=9, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=9, table=TestAcidGuarantees 2024-11-24T22:13:52,886 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37701 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=9 2024-11-24T22:13:52,887 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=9, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=9, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-24T22:13:52,888 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=9, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=9, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-24T22:13:52,888 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=10, ppid=9, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-24T22:13:52,991 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37701 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=9 2024-11-24T22:13:53,021 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40087 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0252adfc27b54a76a05585f3f45bf90, server=bd6592c37bb2,40087,1732486423713 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-24T22:13:53,021 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40087 {}] ipc.CallRunner(138): callId: 51 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36518 deadline: 1732486493011, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0252adfc27b54a76a05585f3f45bf90, server=bd6592c37bb2,40087,1732486423713 2024-11-24T22:13:53,022 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732486426439.f0252adfc27b54a76a05585f3f45bf90., hostname=bd6592c37bb2,40087,1732486423713, seqNum=2 , the old value is region=TestAcidGuarantees,,1732486426439.f0252adfc27b54a76a05585f3f45bf90., hostname=bd6592c37bb2,40087,1732486423713, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0252adfc27b54a76a05585f3f45bf90, server=bd6592c37bb2,40087,1732486423713 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T22:13:53,022 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732486426439.f0252adfc27b54a76a05585f3f45bf90., hostname=bd6592c37bb2,40087,1732486423713, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0252adfc27b54a76a05585f3f45bf90, server=bd6592c37bb2,40087,1732486423713 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T22:13:53,022 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732486426439.f0252adfc27b54a76a05585f3f45bf90., hostname=bd6592c37bb2,40087,1732486423713, seqNum=2 because the exception is null or not the one we care about 2024-11-24T22:13:53,037 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40087 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0252adfc27b54a76a05585f3f45bf90, server=bd6592c37bb2,40087,1732486423713 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-24T22:13:53,037 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40087 {}] ipc.CallRunner(138): callId: 37 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36590 deadline: 1732486493030, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0252adfc27b54a76a05585f3f45bf90, server=bd6592c37bb2,40087,1732486423713 2024-11-24T22:13:53,038 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732486426439.f0252adfc27b54a76a05585f3f45bf90., hostname=bd6592c37bb2,40087,1732486423713, seqNum=2 , the old value is region=TestAcidGuarantees,,1732486426439.f0252adfc27b54a76a05585f3f45bf90., hostname=bd6592c37bb2,40087,1732486423713, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0252adfc27b54a76a05585f3f45bf90, server=bd6592c37bb2,40087,1732486423713 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T22:13:53,038 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732486426439.f0252adfc27b54a76a05585f3f45bf90., hostname=bd6592c37bb2,40087,1732486423713, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0252adfc27b54a76a05585f3f45bf90, server=bd6592c37bb2,40087,1732486423713 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T22:13:53,038 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732486426439.f0252adfc27b54a76a05585f3f45bf90., hostname=bd6592c37bb2,40087,1732486423713, seqNum=2 because the exception is null or not the one we care about 2024-11-24T22:13:53,041 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40087 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0252adfc27b54a76a05585f3f45bf90, server=bd6592c37bb2,40087,1732486423713 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-24T22:13:53,042 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=40087 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=10 2024-11-24T22:13:53,042 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40087 {}] ipc.CallRunner(138): callId: 54 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36562 deadline: 1732486493041, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0252adfc27b54a76a05585f3f45bf90, server=bd6592c37bb2,40087,1732486423713 2024-11-24T22:13:53,042 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bd6592c37bb2:0-2 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732486426439.f0252adfc27b54a76a05585f3f45bf90. 2024-11-24T22:13:53,042 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bd6592c37bb2:0-2 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732486426439.f0252adfc27b54a76a05585f3f45bf90. as already flushing 2024-11-24T22:13:53,042 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732486426439.f0252adfc27b54a76a05585f3f45bf90., hostname=bd6592c37bb2,40087,1732486423713, seqNum=2 , the old value is region=TestAcidGuarantees,,1732486426439.f0252adfc27b54a76a05585f3f45bf90., hostname=bd6592c37bb2,40087,1732486423713, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0252adfc27b54a76a05585f3f45bf90, server=bd6592c37bb2,40087,1732486423713 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T22:13:53,042 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bd6592c37bb2:0-2 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732486426439.f0252adfc27b54a76a05585f3f45bf90. 2024-11-24T22:13:53,042 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732486426439.f0252adfc27b54a76a05585f3f45bf90., hostname=bd6592c37bb2,40087,1732486423713, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0252adfc27b54a76a05585f3f45bf90, server=bd6592c37bb2,40087,1732486423713 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T22:13:53,043 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732486426439.f0252adfc27b54a76a05585f3f45bf90., hostname=bd6592c37bb2,40087,1732486423713, seqNum=2 because the exception is null or not the one we care about 2024-11-24T22:13:53,042 ERROR [RS_FLUSH_OPERATIONS-regionserver/bd6592c37bb2:0-2 {event_type=RS_FLUSH_REGIONS, pid=10}] handler.RSProcedureHandler(58): pid=10 java.io.IOException: Unable to complete flush {ENCODED => f0252adfc27b54a76a05585f3f45bf90, NAME => 'TestAcidGuarantees,,1732486426439.f0252adfc27b54a76a05585f3f45bf90.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T22:13:53,043 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40087 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0252adfc27b54a76a05585f3f45bf90, server=bd6592c37bb2,40087,1732486423713 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-24T22:13:53,043 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bd6592c37bb2:0-2 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=10 java.io.IOException: Unable to complete flush {ENCODED => f0252adfc27b54a76a05585f3f45bf90, NAME => 'TestAcidGuarantees,,1732486426439.f0252adfc27b54a76a05585f3f45bf90.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T22:13:53,043 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40087 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0252adfc27b54a76a05585f3f45bf90, server=bd6592c37bb2,40087,1732486423713 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-24T22:13:53,043 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40087 {}] ipc.CallRunner(138): callId: 53 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36532 deadline: 1732486493041, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0252adfc27b54a76a05585f3f45bf90, server=bd6592c37bb2,40087,1732486423713 2024-11-24T22:13:53,043 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40087 {}] ipc.CallRunner(138): callId: 23 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36548 deadline: 1732486493041, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0252adfc27b54a76a05585f3f45bf90, server=bd6592c37bb2,40087,1732486423713 2024-11-24T22:13:53,044 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732486426439.f0252adfc27b54a76a05585f3f45bf90., hostname=bd6592c37bb2,40087,1732486423713, seqNum=2 , the old value is region=TestAcidGuarantees,,1732486426439.f0252adfc27b54a76a05585f3f45bf90., hostname=bd6592c37bb2,40087,1732486423713, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0252adfc27b54a76a05585f3f45bf90, server=bd6592c37bb2,40087,1732486423713 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T22:13:53,044 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732486426439.f0252adfc27b54a76a05585f3f45bf90., hostname=bd6592c37bb2,40087,1732486423713, seqNum=2 , the old value is region=TestAcidGuarantees,,1732486426439.f0252adfc27b54a76a05585f3f45bf90., hostname=bd6592c37bb2,40087,1732486423713, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0252adfc27b54a76a05585f3f45bf90, server=bd6592c37bb2,40087,1732486423713 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T22:13:53,044 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732486426439.f0252adfc27b54a76a05585f3f45bf90., hostname=bd6592c37bb2,40087,1732486423713, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0252adfc27b54a76a05585f3f45bf90, server=bd6592c37bb2,40087,1732486423713 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T22:13:53,044 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732486426439.f0252adfc27b54a76a05585f3f45bf90., hostname=bd6592c37bb2,40087,1732486423713, seqNum=2 because the exception is null or not the one we care about 2024-11-24T22:13:53,044 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732486426439.f0252adfc27b54a76a05585f3f45bf90., hostname=bd6592c37bb2,40087,1732486423713, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0252adfc27b54a76a05585f3f45bf90, server=bd6592c37bb2,40087,1732486423713 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T22:13:53,044 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732486426439.f0252adfc27b54a76a05585f3f45bf90., hostname=bd6592c37bb2,40087,1732486423713, seqNum=2 because the exception is null or not the one we care about 2024-11-24T22:13:53,044 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37701 {}] master.HMaster(4177): Remote procedure failed, pid=10 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => f0252adfc27b54a76a05585f3f45bf90, NAME => 'TestAcidGuarantees,,1732486426439.f0252adfc27b54a76a05585f3f45bf90.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => f0252adfc27b54a76a05585f3f45bf90, NAME => 'TestAcidGuarantees,,1732486426439.f0252adfc27b54a76a05585f3f45bf90.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T22:13:53,197 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=40087 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=10 2024-11-24T22:13:53,197 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bd6592c37bb2:0-0 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732486426439.f0252adfc27b54a76a05585f3f45bf90. 2024-11-24T22:13:53,197 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bd6592c37bb2:0-0 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732486426439.f0252adfc27b54a76a05585f3f45bf90. as already flushing 2024-11-24T22:13:53,198 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bd6592c37bb2:0-0 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732486426439.f0252adfc27b54a76a05585f3f45bf90. 2024-11-24T22:13:53,198 ERROR [RS_FLUSH_OPERATIONS-regionserver/bd6592c37bb2:0-0 {event_type=RS_FLUSH_REGIONS, pid=10}] handler.RSProcedureHandler(58): pid=10 java.io.IOException: Unable to complete flush {ENCODED => f0252adfc27b54a76a05585f3f45bf90, NAME => 'TestAcidGuarantees,,1732486426439.f0252adfc27b54a76a05585f3f45bf90.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T22:13:53,198 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bd6592c37bb2:0-0 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=10 java.io.IOException: Unable to complete flush {ENCODED => f0252adfc27b54a76a05585f3f45bf90, NAME => 'TestAcidGuarantees,,1732486426439.f0252adfc27b54a76a05585f3f45bf90.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T22:13:53,199 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37701 {}] master.HMaster(4177): Remote procedure failed, pid=10 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => f0252adfc27b54a76a05585f3f45bf90, NAME => 'TestAcidGuarantees,,1732486426439.f0252adfc27b54a76a05585f3f45bf90.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => f0252adfc27b54a76a05585f3f45bf90, NAME => 'TestAcidGuarantees,,1732486426439.f0252adfc27b54a76a05585f3f45bf90.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T22:13:53,201 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37701 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=9 2024-11-24T22:13:53,234 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=38.02 KB at sequenceid=157 (bloomFilter=true), to=hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/f0252adfc27b54a76a05585f3f45bf90/.tmp/B/dc390cb315b24517a00c5d625773456e 2024-11-24T22:13:53,260 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/f0252adfc27b54a76a05585f3f45bf90/.tmp/C/85c936947bd84c1e8e5da34b16ed5bae is 50, key is test_row_0/C:col10/1732486432344/Put/seqid=0 2024-11-24T22:13:53,308 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45929 is added to blk_1073741854_1030 (size=12151) 2024-11-24T22:13:53,309 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=38.02 KB at sequenceid=157 (bloomFilter=true), to=hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/f0252adfc27b54a76a05585f3f45bf90/.tmp/C/85c936947bd84c1e8e5da34b16ed5bae 2024-11-24T22:13:53,321 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/f0252adfc27b54a76a05585f3f45bf90/.tmp/A/ab0287723e304b5ba215650833ca6579 as hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/f0252adfc27b54a76a05585f3f45bf90/A/ab0287723e304b5ba215650833ca6579 2024-11-24T22:13:53,333 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/f0252adfc27b54a76a05585f3f45bf90/A/ab0287723e304b5ba215650833ca6579, entries=150, sequenceid=157, filesize=11.9 K 2024-11-24T22:13:53,336 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/f0252adfc27b54a76a05585f3f45bf90/.tmp/B/dc390cb315b24517a00c5d625773456e as hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/f0252adfc27b54a76a05585f3f45bf90/B/dc390cb315b24517a00c5d625773456e 2024-11-24T22:13:53,350 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/f0252adfc27b54a76a05585f3f45bf90/B/dc390cb315b24517a00c5d625773456e, entries=150, sequenceid=157, filesize=11.9 K 2024-11-24T22:13:53,351 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/f0252adfc27b54a76a05585f3f45bf90/.tmp/C/85c936947bd84c1e8e5da34b16ed5bae as hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/f0252adfc27b54a76a05585f3f45bf90/C/85c936947bd84c1e8e5da34b16ed5bae 2024-11-24T22:13:53,353 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=40087 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=10 2024-11-24T22:13:53,353 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bd6592c37bb2:0-1 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732486426439.f0252adfc27b54a76a05585f3f45bf90. 2024-11-24T22:13:53,353 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bd6592c37bb2:0-1 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732486426439.f0252adfc27b54a76a05585f3f45bf90. as already flushing 2024-11-24T22:13:53,353 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bd6592c37bb2:0-1 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732486426439.f0252adfc27b54a76a05585f3f45bf90. 2024-11-24T22:13:53,354 ERROR [RS_FLUSH_OPERATIONS-regionserver/bd6592c37bb2:0-1 {event_type=RS_FLUSH_REGIONS, pid=10}] handler.RSProcedureHandler(58): pid=10 java.io.IOException: Unable to complete flush {ENCODED => f0252adfc27b54a76a05585f3f45bf90, NAME => 'TestAcidGuarantees,,1732486426439.f0252adfc27b54a76a05585f3f45bf90.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T22:13:53,354 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bd6592c37bb2:0-1 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=10 java.io.IOException: Unable to complete flush {ENCODED => f0252adfc27b54a76a05585f3f45bf90, NAME => 'TestAcidGuarantees,,1732486426439.f0252adfc27b54a76a05585f3f45bf90.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T22:13:53,355 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37701 {}] master.HMaster(4177): Remote procedure failed, pid=10 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => f0252adfc27b54a76a05585f3f45bf90, NAME => 'TestAcidGuarantees,,1732486426439.f0252adfc27b54a76a05585f3f45bf90.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => f0252adfc27b54a76a05585f3f45bf90, NAME => 'TestAcidGuarantees,,1732486426439.f0252adfc27b54a76a05585f3f45bf90.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T22:13:53,375 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/f0252adfc27b54a76a05585f3f45bf90/C/85c936947bd84c1e8e5da34b16ed5bae, entries=150, sequenceid=157, filesize=11.9 K 2024-11-24T22:13:53,377 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~114.05 KB/116790, heapSize ~339.38 KB/347520, currentSize=60.38 KB/61830 for f0252adfc27b54a76a05585f3f45bf90 in 1033ms, sequenceid=157, compaction requested=true 2024-11-24T22:13:53,377 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for f0252adfc27b54a76a05585f3f45bf90: 2024-11-24T22:13:53,377 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store f0252adfc27b54a76a05585f3f45bf90:A, priority=-2147483648, current under compaction store size is 1 2024-11-24T22:13:53,378 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-24T22:13:53,378 DEBUG [RS:0;bd6592c37bb2:40087-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-24T22:13:53,378 DEBUG [RS:0;bd6592c37bb2:40087-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-24T22:13:53,378 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store f0252adfc27b54a76a05585f3f45bf90:B, priority=-2147483648, current under compaction store size is 2 2024-11-24T22:13:53,378 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-24T22:13:53,378 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store f0252adfc27b54a76a05585f3f45bf90:C, priority=-2147483648, current under compaction store size is 3 2024-11-24T22:13:53,378 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-24T22:13:53,381 DEBUG [RS:0;bd6592c37bb2:40087-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36356 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-24T22:13:53,381 DEBUG [RS:0;bd6592c37bb2:40087-longCompactions-0 {}] regionserver.HStore(1541): f0252adfc27b54a76a05585f3f45bf90/B is initiating minor compaction (all files) 2024-11-24T22:13:53,381 INFO [RS:0;bd6592c37bb2:40087-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of f0252adfc27b54a76a05585f3f45bf90/B in TestAcidGuarantees,,1732486426439.f0252adfc27b54a76a05585f3f45bf90. 2024-11-24T22:13:53,381 INFO [RS:0;bd6592c37bb2:40087-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/f0252adfc27b54a76a05585f3f45bf90/B/6cdf2f3301bb413f9d1e2d9b0160fee3, hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/f0252adfc27b54a76a05585f3f45bf90/B/f77f1f3e41f74085b82344614b253a1f, hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/f0252adfc27b54a76a05585f3f45bf90/B/dc390cb315b24517a00c5d625773456e] into tmpdir=hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/f0252adfc27b54a76a05585f3f45bf90/.tmp, totalSize=35.5 K 2024-11-24T22:13:53,381 DEBUG [RS:0;bd6592c37bb2:40087-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36356 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-24T22:13:53,382 DEBUG [RS:0;bd6592c37bb2:40087-shortCompactions-0 {}] regionserver.HStore(1541): f0252adfc27b54a76a05585f3f45bf90/A is initiating minor compaction (all files) 2024-11-24T22:13:53,382 INFO [RS:0;bd6592c37bb2:40087-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of f0252adfc27b54a76a05585f3f45bf90/A in TestAcidGuarantees,,1732486426439.f0252adfc27b54a76a05585f3f45bf90. 2024-11-24T22:13:53,382 INFO [RS:0;bd6592c37bb2:40087-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/f0252adfc27b54a76a05585f3f45bf90/A/0a2765b1e92043a9b4d3fd87edcc95cd, hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/f0252adfc27b54a76a05585f3f45bf90/A/705ad699596346dba6cf6781c5011618, hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/f0252adfc27b54a76a05585f3f45bf90/A/ab0287723e304b5ba215650833ca6579] into tmpdir=hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/f0252adfc27b54a76a05585f3f45bf90/.tmp, totalSize=35.5 K 2024-11-24T22:13:53,382 DEBUG [RS:0;bd6592c37bb2:40087-longCompactions-0 {}] compactions.Compactor(225): Compacting 6cdf2f3301bb413f9d1e2d9b0160fee3, keycount=150, bloomtype=ROW, size=11.8 K, encoding=NONE, compression=NONE, seqNum=120, earliestPutTs=1732486429896 2024-11-24T22:13:53,383 DEBUG [RS:0;bd6592c37bb2:40087-shortCompactions-0 {}] compactions.Compactor(225): Compacting 0a2765b1e92043a9b4d3fd87edcc95cd, keycount=150, bloomtype=ROW, size=11.8 K, encoding=NONE, compression=NONE, seqNum=120, earliestPutTs=1732486429896 2024-11-24T22:13:53,384 DEBUG [RS:0;bd6592c37bb2:40087-shortCompactions-0 {}] compactions.Compactor(225): Compacting 705ad699596346dba6cf6781c5011618, keycount=150, bloomtype=ROW, size=11.8 K, encoding=NONE, compression=NONE, seqNum=134, earliestPutTs=1732486429955 2024-11-24T22:13:53,385 DEBUG [RS:0;bd6592c37bb2:40087-longCompactions-0 {}] compactions.Compactor(225): Compacting f77f1f3e41f74085b82344614b253a1f, keycount=150, bloomtype=ROW, size=11.8 K, encoding=NONE, compression=NONE, seqNum=134, earliestPutTs=1732486429955 2024-11-24T22:13:53,385 DEBUG [RS:0;bd6592c37bb2:40087-shortCompactions-0 {}] compactions.Compactor(225): Compacting ab0287723e304b5ba215650833ca6579, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=157, earliestPutTs=1732486431186 2024-11-24T22:13:53,386 DEBUG [RS:0;bd6592c37bb2:40087-longCompactions-0 {}] compactions.Compactor(225): Compacting dc390cb315b24517a00c5d625773456e, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=157, earliestPutTs=1732486431186 2024-11-24T22:13:53,406 INFO [RS:0;bd6592c37bb2:40087-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): f0252adfc27b54a76a05585f3f45bf90#A#compaction#18 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-24T22:13:53,407 DEBUG [RS:0;bd6592c37bb2:40087-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/f0252adfc27b54a76a05585f3f45bf90/.tmp/A/c8cf77fc903c408d900988a3a2b6a93c is 50, key is test_row_0/A:col10/1732486432344/Put/seqid=0 2024-11-24T22:13:53,411 INFO [RS:0;bd6592c37bb2:40087-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): f0252adfc27b54a76a05585f3f45bf90#B#compaction#19 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-24T22:13:53,412 DEBUG [RS:0;bd6592c37bb2:40087-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/f0252adfc27b54a76a05585f3f45bf90/.tmp/B/be5ac8db548f4f02af9d4afa54364feb is 50, key is test_row_0/B:col10/1732486432344/Put/seqid=0 2024-11-24T22:13:53,413 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_hbase_table_meta 2024-11-24T22:13:53,414 INFO [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=TableRequests_Namespace_hbase_table_meta Metrics about Tables on a single HBase RegionServer 2024-11-24T22:13:53,415 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=Coprocessor.Region.CP_org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint 2024-11-24T22:13:53,415 INFO [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=Coprocessor.Region.CP_org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint Metrics about HBase RegionObservers 2024-11-24T22:13:53,415 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_master_table_store 2024-11-24T22:13:53,415 INFO [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=TableRequests_Namespace_master_table_store Metrics about Tables on a single HBase RegionServer 2024-11-24T22:13:53,416 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_default_table_TestAcidGuarantees 2024-11-24T22:13:53,416 INFO [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=TableRequests_Namespace_default_table_TestAcidGuarantees Metrics about Tables on a single HBase RegionServer 2024-11-24T22:13:53,434 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45929 is added to blk_1073741855_1031 (size=12357) 2024-11-24T22:13:53,434 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45929 is added to blk_1073741856_1032 (size=12357) 2024-11-24T22:13:53,507 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=40087 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=10 2024-11-24T22:13:53,508 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bd6592c37bb2:0-2 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732486426439.f0252adfc27b54a76a05585f3f45bf90. 2024-11-24T22:13:53,508 INFO [RS_FLUSH_OPERATIONS-regionserver/bd6592c37bb2:0-2 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.HRegion(2902): Flushing f0252adfc27b54a76a05585f3f45bf90 3/3 column families, dataSize=60.38 KB heapSize=180.05 KB 2024-11-24T22:13:53,509 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bd6592c37bb2:0-2 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.CompactingMemStore(205): FLUSHING TO DISK f0252adfc27b54a76a05585f3f45bf90, store=A 2024-11-24T22:13:53,509 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bd6592c37bb2:0-2 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-24T22:13:53,509 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bd6592c37bb2:0-2 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.CompactingMemStore(205): FLUSHING TO DISK f0252adfc27b54a76a05585f3f45bf90, store=B 2024-11-24T22:13:53,509 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bd6592c37bb2:0-2 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-24T22:13:53,509 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bd6592c37bb2:0-2 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.CompactingMemStore(205): FLUSHING TO DISK f0252adfc27b54a76a05585f3f45bf90, store=C 2024-11-24T22:13:53,509 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bd6592c37bb2:0-2 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-24T22:13:53,511 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37701 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=9 2024-11-24T22:13:53,520 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bd6592c37bb2:0-2 {event_type=RS_FLUSH_REGIONS, pid=10}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/f0252adfc27b54a76a05585f3f45bf90/.tmp/A/b146b26e6c524e5ca691fb2961fc5ffc is 50, key is test_row_0/A:col10/1732486432365/Put/seqid=0 2024-11-24T22:13:53,540 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45929 is added to blk_1073741857_1033 (size=12151) 2024-11-24T22:13:53,543 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40087 {}] regionserver.HRegion(8855): Flush requested on f0252adfc27b54a76a05585f3f45bf90 2024-11-24T22:13:53,543 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732486426439.f0252adfc27b54a76a05585f3f45bf90. as already flushing 2024-11-24T22:13:53,545 INFO [RS_FLUSH_OPERATIONS-regionserver/bd6592c37bb2:0-2 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=169 (bloomFilter=true), to=hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/f0252adfc27b54a76a05585f3f45bf90/.tmp/A/b146b26e6c524e5ca691fb2961fc5ffc 2024-11-24T22:13:53,567 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bd6592c37bb2:0-2 {event_type=RS_FLUSH_REGIONS, pid=10}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/f0252adfc27b54a76a05585f3f45bf90/.tmp/B/0f8a8c8188ae448487505488e234be20 is 50, key is test_row_0/B:col10/1732486432365/Put/seqid=0 2024-11-24T22:13:53,575 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45929 is added to blk_1073741858_1034 (size=12151) 2024-11-24T22:13:53,594 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40087 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0252adfc27b54a76a05585f3f45bf90, server=bd6592c37bb2,40087,1732486423713 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-24T22:13:53,594 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40087 {}] ipc.CallRunner(138): callId: 41 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36590 deadline: 1732486493584, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0252adfc27b54a76a05585f3f45bf90, server=bd6592c37bb2,40087,1732486423713 2024-11-24T22:13:53,595 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40087 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0252adfc27b54a76a05585f3f45bf90, server=bd6592c37bb2,40087,1732486423713 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-24T22:13:53,595 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732486426439.f0252adfc27b54a76a05585f3f45bf90., hostname=bd6592c37bb2,40087,1732486423713, seqNum=2 , the old value is region=TestAcidGuarantees,,1732486426439.f0252adfc27b54a76a05585f3f45bf90., hostname=bd6592c37bb2,40087,1732486423713, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0252adfc27b54a76a05585f3f45bf90, server=bd6592c37bb2,40087,1732486423713 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T22:13:53,595 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40087 {}] ipc.CallRunner(138): callId: 27 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36548 deadline: 1732486493584, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0252adfc27b54a76a05585f3f45bf90, server=bd6592c37bb2,40087,1732486423713 2024-11-24T22:13:53,595 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732486426439.f0252adfc27b54a76a05585f3f45bf90., hostname=bd6592c37bb2,40087,1732486423713, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0252adfc27b54a76a05585f3f45bf90, server=bd6592c37bb2,40087,1732486423713 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T22:13:53,595 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732486426439.f0252adfc27b54a76a05585f3f45bf90., hostname=bd6592c37bb2,40087,1732486423713, seqNum=2 because the exception is null or not the one we care about 2024-11-24T22:13:53,595 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732486426439.f0252adfc27b54a76a05585f3f45bf90., hostname=bd6592c37bb2,40087,1732486423713, seqNum=2 , the old value is region=TestAcidGuarantees,,1732486426439.f0252adfc27b54a76a05585f3f45bf90., hostname=bd6592c37bb2,40087,1732486423713, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0252adfc27b54a76a05585f3f45bf90, server=bd6592c37bb2,40087,1732486423713 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T22:13:53,595 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732486426439.f0252adfc27b54a76a05585f3f45bf90., hostname=bd6592c37bb2,40087,1732486423713, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0252adfc27b54a76a05585f3f45bf90, server=bd6592c37bb2,40087,1732486423713 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T22:13:53,595 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732486426439.f0252adfc27b54a76a05585f3f45bf90., hostname=bd6592c37bb2,40087,1732486423713, seqNum=2 because the exception is null or not the one we care about 2024-11-24T22:13:53,599 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40087 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0252adfc27b54a76a05585f3f45bf90, server=bd6592c37bb2,40087,1732486423713 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-24T22:13:53,599 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40087 {}] ipc.CallRunner(138): callId: 58 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36562 deadline: 1732486493591, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0252adfc27b54a76a05585f3f45bf90, server=bd6592c37bb2,40087,1732486423713 2024-11-24T22:13:53,600 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732486426439.f0252adfc27b54a76a05585f3f45bf90., hostname=bd6592c37bb2,40087,1732486423713, seqNum=2 , the old value is region=TestAcidGuarantees,,1732486426439.f0252adfc27b54a76a05585f3f45bf90., hostname=bd6592c37bb2,40087,1732486423713, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0252adfc27b54a76a05585f3f45bf90, server=bd6592c37bb2,40087,1732486423713 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T22:13:53,599 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40087 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0252adfc27b54a76a05585f3f45bf90, server=bd6592c37bb2,40087,1732486423713 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-24T22:13:53,600 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732486426439.f0252adfc27b54a76a05585f3f45bf90., hostname=bd6592c37bb2,40087,1732486423713, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0252adfc27b54a76a05585f3f45bf90, server=bd6592c37bb2,40087,1732486423713 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T22:13:53,600 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732486426439.f0252adfc27b54a76a05585f3f45bf90., hostname=bd6592c37bb2,40087,1732486423713, seqNum=2 because the exception is null or not the one we care about 2024-11-24T22:13:53,600 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40087 {}] ipc.CallRunner(138): callId: 57 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36532 deadline: 1732486493592, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0252adfc27b54a76a05585f3f45bf90, server=bd6592c37bb2,40087,1732486423713 2024-11-24T22:13:53,600 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40087 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0252adfc27b54a76a05585f3f45bf90, server=bd6592c37bb2,40087,1732486423713 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-24T22:13:53,600 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732486426439.f0252adfc27b54a76a05585f3f45bf90., hostname=bd6592c37bb2,40087,1732486423713, seqNum=2 , the old value is region=TestAcidGuarantees,,1732486426439.f0252adfc27b54a76a05585f3f45bf90., hostname=bd6592c37bb2,40087,1732486423713, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0252adfc27b54a76a05585f3f45bf90, server=bd6592c37bb2,40087,1732486423713 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T22:13:53,600 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732486426439.f0252adfc27b54a76a05585f3f45bf90., hostname=bd6592c37bb2,40087,1732486423713, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0252adfc27b54a76a05585f3f45bf90, server=bd6592c37bb2,40087,1732486423713 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T22:13:53,600 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40087 {}] ipc.CallRunner(138): callId: 57 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36518 deadline: 1732486493594, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0252adfc27b54a76a05585f3f45bf90, server=bd6592c37bb2,40087,1732486423713 2024-11-24T22:13:53,600 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732486426439.f0252adfc27b54a76a05585f3f45bf90., hostname=bd6592c37bb2,40087,1732486423713, seqNum=2 because the exception is null or not the one we care about 2024-11-24T22:13:53,601 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732486426439.f0252adfc27b54a76a05585f3f45bf90., hostname=bd6592c37bb2,40087,1732486423713, seqNum=2 , the old value is region=TestAcidGuarantees,,1732486426439.f0252adfc27b54a76a05585f3f45bf90., hostname=bd6592c37bb2,40087,1732486423713, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0252adfc27b54a76a05585f3f45bf90, server=bd6592c37bb2,40087,1732486423713 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T22:13:53,601 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732486426439.f0252adfc27b54a76a05585f3f45bf90., hostname=bd6592c37bb2,40087,1732486423713, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0252adfc27b54a76a05585f3f45bf90, server=bd6592c37bb2,40087,1732486423713 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T22:13:53,601 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732486426439.f0252adfc27b54a76a05585f3f45bf90., hostname=bd6592c37bb2,40087,1732486423713, seqNum=2 because the exception is null or not the one we care about 2024-11-24T22:13:53,709 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40087 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0252adfc27b54a76a05585f3f45bf90, server=bd6592c37bb2,40087,1732486423713 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-24T22:13:53,709 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40087 {}] ipc.CallRunner(138): callId: 42 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36590 deadline: 1732486493701, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0252adfc27b54a76a05585f3f45bf90, server=bd6592c37bb2,40087,1732486423713 2024-11-24T22:13:53,710 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40087 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0252adfc27b54a76a05585f3f45bf90, server=bd6592c37bb2,40087,1732486423713 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-24T22:13:53,710 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732486426439.f0252adfc27b54a76a05585f3f45bf90., hostname=bd6592c37bb2,40087,1732486423713, seqNum=2 , the old value is region=TestAcidGuarantees,,1732486426439.f0252adfc27b54a76a05585f3f45bf90., hostname=bd6592c37bb2,40087,1732486423713, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0252adfc27b54a76a05585f3f45bf90, server=bd6592c37bb2,40087,1732486423713 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T22:13:53,710 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732486426439.f0252adfc27b54a76a05585f3f45bf90., hostname=bd6592c37bb2,40087,1732486423713, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0252adfc27b54a76a05585f3f45bf90, server=bd6592c37bb2,40087,1732486423713 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T22:13:53,710 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40087 {}] ipc.CallRunner(138): callId: 28 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36548 deadline: 1732486493701, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0252adfc27b54a76a05585f3f45bf90, server=bd6592c37bb2,40087,1732486423713 2024-11-24T22:13:53,710 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732486426439.f0252adfc27b54a76a05585f3f45bf90., hostname=bd6592c37bb2,40087,1732486423713, seqNum=2 because the exception is null or not the one we care about 2024-11-24T22:13:53,711 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732486426439.f0252adfc27b54a76a05585f3f45bf90., hostname=bd6592c37bb2,40087,1732486423713, seqNum=2 , the old value is region=TestAcidGuarantees,,1732486426439.f0252adfc27b54a76a05585f3f45bf90., hostname=bd6592c37bb2,40087,1732486423713, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0252adfc27b54a76a05585f3f45bf90, server=bd6592c37bb2,40087,1732486423713 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T22:13:53,711 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732486426439.f0252adfc27b54a76a05585f3f45bf90., hostname=bd6592c37bb2,40087,1732486423713, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0252adfc27b54a76a05585f3f45bf90, server=bd6592c37bb2,40087,1732486423713 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T22:13:53,711 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732486426439.f0252adfc27b54a76a05585f3f45bf90., hostname=bd6592c37bb2,40087,1732486423713, seqNum=2 because the exception is null or not the one we care about 2024-11-24T22:13:53,713 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40087 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0252adfc27b54a76a05585f3f45bf90, server=bd6592c37bb2,40087,1732486423713 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-24T22:13:53,713 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40087 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0252adfc27b54a76a05585f3f45bf90, server=bd6592c37bb2,40087,1732486423713 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-24T22:13:53,714 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40087 {}] ipc.CallRunner(138): callId: 58 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36518 deadline: 1732486493712, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0252adfc27b54a76a05585f3f45bf90, server=bd6592c37bb2,40087,1732486423713 2024-11-24T22:13:53,714 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40087 {}] ipc.CallRunner(138): callId: 59 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36562 deadline: 1732486493713, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0252adfc27b54a76a05585f3f45bf90, server=bd6592c37bb2,40087,1732486423713 2024-11-24T22:13:53,714 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732486426439.f0252adfc27b54a76a05585f3f45bf90., hostname=bd6592c37bb2,40087,1732486423713, seqNum=2 , the old value is region=TestAcidGuarantees,,1732486426439.f0252adfc27b54a76a05585f3f45bf90., hostname=bd6592c37bb2,40087,1732486423713, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0252adfc27b54a76a05585f3f45bf90, server=bd6592c37bb2,40087,1732486423713 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T22:13:53,714 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732486426439.f0252adfc27b54a76a05585f3f45bf90., hostname=bd6592c37bb2,40087,1732486423713, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0252adfc27b54a76a05585f3f45bf90, server=bd6592c37bb2,40087,1732486423713 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T22:13:53,715 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732486426439.f0252adfc27b54a76a05585f3f45bf90., hostname=bd6592c37bb2,40087,1732486423713, seqNum=2 because the exception is null or not the one we care about 2024-11-24T22:13:53,715 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732486426439.f0252adfc27b54a76a05585f3f45bf90., hostname=bd6592c37bb2,40087,1732486423713, seqNum=2 , the old value is region=TestAcidGuarantees,,1732486426439.f0252adfc27b54a76a05585f3f45bf90., hostname=bd6592c37bb2,40087,1732486423713, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0252adfc27b54a76a05585f3f45bf90, server=bd6592c37bb2,40087,1732486423713 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T22:13:53,715 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732486426439.f0252adfc27b54a76a05585f3f45bf90., hostname=bd6592c37bb2,40087,1732486423713, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0252adfc27b54a76a05585f3f45bf90, server=bd6592c37bb2,40087,1732486423713 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T22:13:53,715 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732486426439.f0252adfc27b54a76a05585f3f45bf90., hostname=bd6592c37bb2,40087,1732486423713, seqNum=2 because the exception is null or not the one we care about 2024-11-24T22:13:53,720 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40087 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0252adfc27b54a76a05585f3f45bf90, server=bd6592c37bb2,40087,1732486423713 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-24T22:13:53,720 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40087 {}] ipc.CallRunner(138): callId: 58 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36532 deadline: 1732486493716, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0252adfc27b54a76a05585f3f45bf90, server=bd6592c37bb2,40087,1732486423713 2024-11-24T22:13:53,721 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732486426439.f0252adfc27b54a76a05585f3f45bf90., hostname=bd6592c37bb2,40087,1732486423713, seqNum=2 , the old value is region=TestAcidGuarantees,,1732486426439.f0252adfc27b54a76a05585f3f45bf90., hostname=bd6592c37bb2,40087,1732486423713, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0252adfc27b54a76a05585f3f45bf90, server=bd6592c37bb2,40087,1732486423713 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T22:13:53,721 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732486426439.f0252adfc27b54a76a05585f3f45bf90., hostname=bd6592c37bb2,40087,1732486423713, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0252adfc27b54a76a05585f3f45bf90, server=bd6592c37bb2,40087,1732486423713 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T22:13:53,721 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732486426439.f0252adfc27b54a76a05585f3f45bf90., hostname=bd6592c37bb2,40087,1732486423713, seqNum=2 because the exception is null or not the one we care about 2024-11-24T22:13:53,842 DEBUG [RS:0;bd6592c37bb2:40087-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/f0252adfc27b54a76a05585f3f45bf90/.tmp/A/c8cf77fc903c408d900988a3a2b6a93c as hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/f0252adfc27b54a76a05585f3f45bf90/A/c8cf77fc903c408d900988a3a2b6a93c 2024-11-24T22:13:53,846 DEBUG [RS:0;bd6592c37bb2:40087-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/f0252adfc27b54a76a05585f3f45bf90/.tmp/B/be5ac8db548f4f02af9d4afa54364feb as hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/f0252adfc27b54a76a05585f3f45bf90/B/be5ac8db548f4f02af9d4afa54364feb 2024-11-24T22:13:53,856 INFO [RS:0;bd6592c37bb2:40087-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in f0252adfc27b54a76a05585f3f45bf90/A of f0252adfc27b54a76a05585f3f45bf90 into c8cf77fc903c408d900988a3a2b6a93c(size=12.1 K), total size for store is 12.1 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-24T22:13:53,856 DEBUG [RS:0;bd6592c37bb2:40087-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for f0252adfc27b54a76a05585f3f45bf90: 2024-11-24T22:13:53,856 INFO [RS:0;bd6592c37bb2:40087-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732486426439.f0252adfc27b54a76a05585f3f45bf90., storeName=f0252adfc27b54a76a05585f3f45bf90/A, priority=13, startTime=1732486433377; duration=0sec 2024-11-24T22:13:53,857 DEBUG [RS:0;bd6592c37bb2:40087-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-24T22:13:53,857 DEBUG [RS:0;bd6592c37bb2:40087-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: f0252adfc27b54a76a05585f3f45bf90:A 2024-11-24T22:13:53,857 DEBUG [RS:0;bd6592c37bb2:40087-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-24T22:13:53,860 DEBUG [RS:0;bd6592c37bb2:40087-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36356 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-24T22:13:53,861 DEBUG [RS:0;bd6592c37bb2:40087-shortCompactions-0 {}] regionserver.HStore(1541): f0252adfc27b54a76a05585f3f45bf90/C is initiating minor compaction (all files) 2024-11-24T22:13:53,861 INFO [RS:0;bd6592c37bb2:40087-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of f0252adfc27b54a76a05585f3f45bf90/C in TestAcidGuarantees,,1732486426439.f0252adfc27b54a76a05585f3f45bf90. 2024-11-24T22:13:53,861 INFO [RS:0;bd6592c37bb2:40087-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/f0252adfc27b54a76a05585f3f45bf90/C/88c916dc5d6b43af9776f182ebf2959c, hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/f0252adfc27b54a76a05585f3f45bf90/C/7fadf24fe8a7459ea252bf9e20a561a7, hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/f0252adfc27b54a76a05585f3f45bf90/C/85c936947bd84c1e8e5da34b16ed5bae] into tmpdir=hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/f0252adfc27b54a76a05585f3f45bf90/.tmp, totalSize=35.5 K 2024-11-24T22:13:53,862 INFO [RS:0;bd6592c37bb2:40087-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in f0252adfc27b54a76a05585f3f45bf90/B of f0252adfc27b54a76a05585f3f45bf90 into be5ac8db548f4f02af9d4afa54364feb(size=12.1 K), total size for store is 12.1 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-24T22:13:53,862 DEBUG [RS:0;bd6592c37bb2:40087-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for f0252adfc27b54a76a05585f3f45bf90: 2024-11-24T22:13:53,862 INFO [RS:0;bd6592c37bb2:40087-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732486426439.f0252adfc27b54a76a05585f3f45bf90., storeName=f0252adfc27b54a76a05585f3f45bf90/B, priority=13, startTime=1732486433378; duration=0sec 2024-11-24T22:13:53,862 DEBUG [RS:0;bd6592c37bb2:40087-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-24T22:13:53,862 DEBUG [RS:0;bd6592c37bb2:40087-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: f0252adfc27b54a76a05585f3f45bf90:B 2024-11-24T22:13:53,862 DEBUG [RS:0;bd6592c37bb2:40087-shortCompactions-0 {}] compactions.Compactor(225): Compacting 88c916dc5d6b43af9776f182ebf2959c, keycount=150, bloomtype=ROW, size=11.8 K, encoding=NONE, compression=NONE, seqNum=120, earliestPutTs=1732486429896 2024-11-24T22:13:53,863 DEBUG [RS:0;bd6592c37bb2:40087-shortCompactions-0 {}] compactions.Compactor(225): Compacting 7fadf24fe8a7459ea252bf9e20a561a7, keycount=150, bloomtype=ROW, size=11.8 K, encoding=NONE, compression=NONE, seqNum=134, earliestPutTs=1732486429955 2024-11-24T22:13:53,863 DEBUG [RS:0;bd6592c37bb2:40087-shortCompactions-0 {}] compactions.Compactor(225): Compacting 85c936947bd84c1e8e5da34b16ed5bae, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=157, earliestPutTs=1732486431186 2024-11-24T22:13:53,880 INFO [RS:0;bd6592c37bb2:40087-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): f0252adfc27b54a76a05585f3f45bf90#C#compaction#22 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-24T22:13:53,881 DEBUG [RS:0;bd6592c37bb2:40087-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/f0252adfc27b54a76a05585f3f45bf90/.tmp/C/bc92118f30b44a4995b050c164edd61f is 50, key is test_row_0/C:col10/1732486432344/Put/seqid=0 2024-11-24T22:13:53,896 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45929 is added to blk_1073741859_1035 (size=12357) 2024-11-24T22:13:53,910 DEBUG [RS:0;bd6592c37bb2:40087-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/f0252adfc27b54a76a05585f3f45bf90/.tmp/C/bc92118f30b44a4995b050c164edd61f as hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/f0252adfc27b54a76a05585f3f45bf90/C/bc92118f30b44a4995b050c164edd61f 2024-11-24T22:13:53,923 INFO [RS:0;bd6592c37bb2:40087-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in f0252adfc27b54a76a05585f3f45bf90/C of f0252adfc27b54a76a05585f3f45bf90 into bc92118f30b44a4995b050c164edd61f(size=12.1 K), total size for store is 12.1 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-24T22:13:53,924 DEBUG [RS:0;bd6592c37bb2:40087-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for f0252adfc27b54a76a05585f3f45bf90: 2024-11-24T22:13:53,924 INFO [RS:0;bd6592c37bb2:40087-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732486426439.f0252adfc27b54a76a05585f3f45bf90., storeName=f0252adfc27b54a76a05585f3f45bf90/C, priority=13, startTime=1732486433378; duration=0sec 2024-11-24T22:13:53,924 DEBUG [RS:0;bd6592c37bb2:40087-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-24T22:13:53,924 DEBUG [RS:0;bd6592c37bb2:40087-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: f0252adfc27b54a76a05585f3f45bf90:C 2024-11-24T22:13:53,926 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40087 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0252adfc27b54a76a05585f3f45bf90, server=bd6592c37bb2,40087,1732486423713 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-24T22:13:53,926 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40087 {}] ipc.CallRunner(138): callId: 43 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36590 deadline: 1732486493921, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0252adfc27b54a76a05585f3f45bf90, server=bd6592c37bb2,40087,1732486423713 2024-11-24T22:13:53,926 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732486426439.f0252adfc27b54a76a05585f3f45bf90., hostname=bd6592c37bb2,40087,1732486423713, seqNum=2 , the old value is region=TestAcidGuarantees,,1732486426439.f0252adfc27b54a76a05585f3f45bf90., hostname=bd6592c37bb2,40087,1732486423713, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0252adfc27b54a76a05585f3f45bf90, server=bd6592c37bb2,40087,1732486423713 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T22:13:53,926 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40087 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0252adfc27b54a76a05585f3f45bf90, server=bd6592c37bb2,40087,1732486423713 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-24T22:13:53,927 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732486426439.f0252adfc27b54a76a05585f3f45bf90., hostname=bd6592c37bb2,40087,1732486423713, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0252adfc27b54a76a05585f3f45bf90, server=bd6592c37bb2,40087,1732486423713 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T22:13:53,927 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732486426439.f0252adfc27b54a76a05585f3f45bf90., hostname=bd6592c37bb2,40087,1732486423713, seqNum=2 because the exception is null or not the one we care about 2024-11-24T22:13:53,927 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40087 {}] ipc.CallRunner(138): callId: 29 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36548 deadline: 1732486493921, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0252adfc27b54a76a05585f3f45bf90, server=bd6592c37bb2,40087,1732486423713 2024-11-24T22:13:53,927 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732486426439.f0252adfc27b54a76a05585f3f45bf90., hostname=bd6592c37bb2,40087,1732486423713, seqNum=2 , the old value is region=TestAcidGuarantees,,1732486426439.f0252adfc27b54a76a05585f3f45bf90., hostname=bd6592c37bb2,40087,1732486423713, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0252adfc27b54a76a05585f3f45bf90, server=bd6592c37bb2,40087,1732486423713 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T22:13:53,927 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732486426439.f0252adfc27b54a76a05585f3f45bf90., hostname=bd6592c37bb2,40087,1732486423713, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0252adfc27b54a76a05585f3f45bf90, server=bd6592c37bb2,40087,1732486423713 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T22:13:53,927 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732486426439.f0252adfc27b54a76a05585f3f45bf90., hostname=bd6592c37bb2,40087,1732486423713, seqNum=2 because the exception is null or not the one we care about 2024-11-24T22:13:53,927 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40087 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0252adfc27b54a76a05585f3f45bf90, server=bd6592c37bb2,40087,1732486423713 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-24T22:13:53,927 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40087 {}] ipc.CallRunner(138): callId: 60 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36562 deadline: 1732486493921, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0252adfc27b54a76a05585f3f45bf90, server=bd6592c37bb2,40087,1732486423713 2024-11-24T22:13:53,928 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732486426439.f0252adfc27b54a76a05585f3f45bf90., hostname=bd6592c37bb2,40087,1732486423713, seqNum=2 , the old value is region=TestAcidGuarantees,,1732486426439.f0252adfc27b54a76a05585f3f45bf90., hostname=bd6592c37bb2,40087,1732486423713, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0252adfc27b54a76a05585f3f45bf90, server=bd6592c37bb2,40087,1732486423713 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T22:13:53,928 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732486426439.f0252adfc27b54a76a05585f3f45bf90., hostname=bd6592c37bb2,40087,1732486423713, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0252adfc27b54a76a05585f3f45bf90, server=bd6592c37bb2,40087,1732486423713 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T22:13:53,928 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732486426439.f0252adfc27b54a76a05585f3f45bf90., hostname=bd6592c37bb2,40087,1732486423713, seqNum=2 because the exception is null or not the one we care about 2024-11-24T22:13:53,928 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40087 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0252adfc27b54a76a05585f3f45bf90, server=bd6592c37bb2,40087,1732486423713 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-24T22:13:53,928 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40087 {}] ipc.CallRunner(138): callId: 59 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36518 deadline: 1732486493921, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0252adfc27b54a76a05585f3f45bf90, server=bd6592c37bb2,40087,1732486423713 2024-11-24T22:13:53,928 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732486426439.f0252adfc27b54a76a05585f3f45bf90., hostname=bd6592c37bb2,40087,1732486423713, seqNum=2 , the old value is region=TestAcidGuarantees,,1732486426439.f0252adfc27b54a76a05585f3f45bf90., hostname=bd6592c37bb2,40087,1732486423713, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0252adfc27b54a76a05585f3f45bf90, server=bd6592c37bb2,40087,1732486423713 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T22:13:53,928 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732486426439.f0252adfc27b54a76a05585f3f45bf90., hostname=bd6592c37bb2,40087,1732486423713, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0252adfc27b54a76a05585f3f45bf90, server=bd6592c37bb2,40087,1732486423713 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T22:13:53,928 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732486426439.f0252adfc27b54a76a05585f3f45bf90., hostname=bd6592c37bb2,40087,1732486423713, seqNum=2 because the exception is null or not the one we care about 2024-11-24T22:13:53,934 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40087 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0252adfc27b54a76a05585f3f45bf90, server=bd6592c37bb2,40087,1732486423713 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-24T22:13:53,934 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40087 {}] ipc.CallRunner(138): callId: 59 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36532 deadline: 1732486493931, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0252adfc27b54a76a05585f3f45bf90, server=bd6592c37bb2,40087,1732486423713 2024-11-24T22:13:53,935 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732486426439.f0252adfc27b54a76a05585f3f45bf90., hostname=bd6592c37bb2,40087,1732486423713, seqNum=2 , the old value is region=TestAcidGuarantees,,1732486426439.f0252adfc27b54a76a05585f3f45bf90., hostname=bd6592c37bb2,40087,1732486423713, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0252adfc27b54a76a05585f3f45bf90, server=bd6592c37bb2,40087,1732486423713 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T22:13:53,935 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732486426439.f0252adfc27b54a76a05585f3f45bf90., hostname=bd6592c37bb2,40087,1732486423713, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0252adfc27b54a76a05585f3f45bf90, server=bd6592c37bb2,40087,1732486423713 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T22:13:53,935 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732486426439.f0252adfc27b54a76a05585f3f45bf90., hostname=bd6592c37bb2,40087,1732486423713, seqNum=2 because the exception is null or not the one we care about 2024-11-24T22:13:53,977 INFO [RS_FLUSH_OPERATIONS-regionserver/bd6592c37bb2:0-2 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=169 (bloomFilter=true), to=hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/f0252adfc27b54a76a05585f3f45bf90/.tmp/B/0f8a8c8188ae448487505488e234be20 2024-11-24T22:13:53,995 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bd6592c37bb2:0-2 {event_type=RS_FLUSH_REGIONS, pid=10}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/f0252adfc27b54a76a05585f3f45bf90/.tmp/C/7bbbf641f1b44d9195ae936a79cb2684 is 50, key is test_row_0/C:col10/1732486432365/Put/seqid=0 2024-11-24T22:13:54,010 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45929 is added to blk_1073741860_1036 (size=12151) 2024-11-24T22:13:54,021 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37701 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=9 2024-11-24T22:13:54,233 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40087 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0252adfc27b54a76a05585f3f45bf90, server=bd6592c37bb2,40087,1732486423713 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-24T22:13:54,233 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40087 {}] ipc.CallRunner(138): callId: 44 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36590 deadline: 1732486494230, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0252adfc27b54a76a05585f3f45bf90, server=bd6592c37bb2,40087,1732486423713 2024-11-24T22:13:54,233 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40087 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0252adfc27b54a76a05585f3f45bf90, server=bd6592c37bb2,40087,1732486423713 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-24T22:13:54,234 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40087 {}] ipc.CallRunner(138): callId: 30 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36548 deadline: 1732486494231, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0252adfc27b54a76a05585f3f45bf90, server=bd6592c37bb2,40087,1732486423713 2024-11-24T22:13:54,234 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732486426439.f0252adfc27b54a76a05585f3f45bf90., hostname=bd6592c37bb2,40087,1732486423713, seqNum=2 , the old value is region=TestAcidGuarantees,,1732486426439.f0252adfc27b54a76a05585f3f45bf90., hostname=bd6592c37bb2,40087,1732486423713, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0252adfc27b54a76a05585f3f45bf90, server=bd6592c37bb2,40087,1732486423713 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T22:13:54,234 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732486426439.f0252adfc27b54a76a05585f3f45bf90., hostname=bd6592c37bb2,40087,1732486423713, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0252adfc27b54a76a05585f3f45bf90, server=bd6592c37bb2,40087,1732486423713 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T22:13:54,234 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732486426439.f0252adfc27b54a76a05585f3f45bf90., hostname=bd6592c37bb2,40087,1732486423713, seqNum=2 because the exception is null or not the one we care about 2024-11-24T22:13:54,236 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732486426439.f0252adfc27b54a76a05585f3f45bf90., hostname=bd6592c37bb2,40087,1732486423713, seqNum=2 , the old value is region=TestAcidGuarantees,,1732486426439.f0252adfc27b54a76a05585f3f45bf90., hostname=bd6592c37bb2,40087,1732486423713, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0252adfc27b54a76a05585f3f45bf90, server=bd6592c37bb2,40087,1732486423713 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T22:13:54,236 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732486426439.f0252adfc27b54a76a05585f3f45bf90., hostname=bd6592c37bb2,40087,1732486423713, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0252adfc27b54a76a05585f3f45bf90, server=bd6592c37bb2,40087,1732486423713 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T22:13:54,236 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732486426439.f0252adfc27b54a76a05585f3f45bf90., hostname=bd6592c37bb2,40087,1732486423713, seqNum=2 because the exception is null or not the one we care about 2024-11-24T22:13:54,244 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40087 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0252adfc27b54a76a05585f3f45bf90, server=bd6592c37bb2,40087,1732486423713 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-24T22:13:54,245 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40087 {}] ipc.CallRunner(138): callId: 61 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36562 deadline: 1732486494241, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0252adfc27b54a76a05585f3f45bf90, server=bd6592c37bb2,40087,1732486423713 2024-11-24T22:13:54,245 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732486426439.f0252adfc27b54a76a05585f3f45bf90., hostname=bd6592c37bb2,40087,1732486423713, seqNum=2 , the old value is region=TestAcidGuarantees,,1732486426439.f0252adfc27b54a76a05585f3f45bf90., hostname=bd6592c37bb2,40087,1732486423713, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0252adfc27b54a76a05585f3f45bf90, server=bd6592c37bb2,40087,1732486423713 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T22:13:54,245 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732486426439.f0252adfc27b54a76a05585f3f45bf90., hostname=bd6592c37bb2,40087,1732486423713, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0252adfc27b54a76a05585f3f45bf90, server=bd6592c37bb2,40087,1732486423713 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T22:13:54,245 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40087 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0252adfc27b54a76a05585f3f45bf90, server=bd6592c37bb2,40087,1732486423713 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-24T22:13:54,245 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732486426439.f0252adfc27b54a76a05585f3f45bf90., hostname=bd6592c37bb2,40087,1732486423713, seqNum=2 because the exception is null or not the one we care about 2024-11-24T22:13:54,245 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40087 {}] ipc.CallRunner(138): callId: 60 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36532 deadline: 1732486494241, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0252adfc27b54a76a05585f3f45bf90, server=bd6592c37bb2,40087,1732486423713 2024-11-24T22:13:54,246 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732486426439.f0252adfc27b54a76a05585f3f45bf90., hostname=bd6592c37bb2,40087,1732486423713, seqNum=2 , the old value is region=TestAcidGuarantees,,1732486426439.f0252adfc27b54a76a05585f3f45bf90., hostname=bd6592c37bb2,40087,1732486423713, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0252adfc27b54a76a05585f3f45bf90, server=bd6592c37bb2,40087,1732486423713 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T22:13:54,246 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732486426439.f0252adfc27b54a76a05585f3f45bf90., hostname=bd6592c37bb2,40087,1732486423713, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0252adfc27b54a76a05585f3f45bf90, server=bd6592c37bb2,40087,1732486423713 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T22:13:54,246 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732486426439.f0252adfc27b54a76a05585f3f45bf90., hostname=bd6592c37bb2,40087,1732486423713, seqNum=2 because the exception is null or not the one we care about 2024-11-24T22:13:54,246 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40087 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0252adfc27b54a76a05585f3f45bf90, server=bd6592c37bb2,40087,1732486423713 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-24T22:13:54,246 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40087 {}] ipc.CallRunner(138): callId: 60 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36518 deadline: 1732486494241, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0252adfc27b54a76a05585f3f45bf90, server=bd6592c37bb2,40087,1732486423713 2024-11-24T22:13:54,246 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732486426439.f0252adfc27b54a76a05585f3f45bf90., hostname=bd6592c37bb2,40087,1732486423713, seqNum=2 , the old value is region=TestAcidGuarantees,,1732486426439.f0252adfc27b54a76a05585f3f45bf90., hostname=bd6592c37bb2,40087,1732486423713, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0252adfc27b54a76a05585f3f45bf90, server=bd6592c37bb2,40087,1732486423713 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T22:13:54,247 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732486426439.f0252adfc27b54a76a05585f3f45bf90., hostname=bd6592c37bb2,40087,1732486423713, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0252adfc27b54a76a05585f3f45bf90, server=bd6592c37bb2,40087,1732486423713 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T22:13:54,247 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732486426439.f0252adfc27b54a76a05585f3f45bf90., hostname=bd6592c37bb2,40087,1732486423713, seqNum=2 because the exception is null or not the one we care about 2024-11-24T22:13:54,412 INFO [RS_FLUSH_OPERATIONS-regionserver/bd6592c37bb2:0-2 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=169 (bloomFilter=true), to=hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/f0252adfc27b54a76a05585f3f45bf90/.tmp/C/7bbbf641f1b44d9195ae936a79cb2684 2024-11-24T22:13:54,422 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bd6592c37bb2:0-2 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/f0252adfc27b54a76a05585f3f45bf90/.tmp/A/b146b26e6c524e5ca691fb2961fc5ffc as hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/f0252adfc27b54a76a05585f3f45bf90/A/b146b26e6c524e5ca691fb2961fc5ffc 2024-11-24T22:13:54,429 INFO [RS_FLUSH_OPERATIONS-regionserver/bd6592c37bb2:0-2 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/f0252adfc27b54a76a05585f3f45bf90/A/b146b26e6c524e5ca691fb2961fc5ffc, entries=150, sequenceid=169, filesize=11.9 K 2024-11-24T22:13:54,431 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bd6592c37bb2:0-2 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/f0252adfc27b54a76a05585f3f45bf90/.tmp/B/0f8a8c8188ae448487505488e234be20 as hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/f0252adfc27b54a76a05585f3f45bf90/B/0f8a8c8188ae448487505488e234be20 2024-11-24T22:13:54,438 INFO [RS_FLUSH_OPERATIONS-regionserver/bd6592c37bb2:0-2 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/f0252adfc27b54a76a05585f3f45bf90/B/0f8a8c8188ae448487505488e234be20, entries=150, sequenceid=169, filesize=11.9 K 2024-11-24T22:13:54,440 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bd6592c37bb2:0-2 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/f0252adfc27b54a76a05585f3f45bf90/.tmp/C/7bbbf641f1b44d9195ae936a79cb2684 as hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/f0252adfc27b54a76a05585f3f45bf90/C/7bbbf641f1b44d9195ae936a79cb2684 2024-11-24T22:13:54,446 INFO [RS_FLUSH_OPERATIONS-regionserver/bd6592c37bb2:0-2 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/f0252adfc27b54a76a05585f3f45bf90/C/7bbbf641f1b44d9195ae936a79cb2684, entries=150, sequenceid=169, filesize=11.9 K 2024-11-24T22:13:54,448 INFO [RS_FLUSH_OPERATIONS-regionserver/bd6592c37bb2:0-2 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.HRegion(3140): Finished flush of dataSize ~60.38 KB/61830, heapSize ~180 KB/184320, currentSize=114.05 KB/116790 for f0252adfc27b54a76a05585f3f45bf90 in 940ms, sequenceid=169, compaction requested=false 2024-11-24T22:13:54,448 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bd6592c37bb2:0-2 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.HRegion(2603): Flush status journal for f0252adfc27b54a76a05585f3f45bf90: 2024-11-24T22:13:54,448 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bd6592c37bb2:0-2 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732486426439.f0252adfc27b54a76a05585f3f45bf90. 2024-11-24T22:13:54,448 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bd6592c37bb2:0-2 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=10 2024-11-24T22:13:54,449 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37701 {}] master.HMaster(4169): Remote procedure done, pid=10 2024-11-24T22:13:54,457 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=10, resume processing ppid=9 2024-11-24T22:13:54,457 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=10, ppid=9, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.5630 sec 2024-11-24T22:13:54,462 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=9, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=9, table=TestAcidGuarantees in 1.5770 sec 2024-11-24T22:13:54,747 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40087 {}] regionserver.HRegion(8855): Flush requested on f0252adfc27b54a76a05585f3f45bf90 2024-11-24T22:13:54,747 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/bd6592c37bb2:0-6 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1732486433576, max timestamp=1732486434743Num uniques 859; 2024-11-24T22:13:54,747 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/bd6592c37bb2:0-7 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1732486433576, max timestamp=1732486434743Num uniques 859; 2024-11-24T22:13:54,747 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing f0252adfc27b54a76a05585f3f45bf90 3/3 column families, dataSize=120.76 KB heapSize=360.05 KB 2024-11-24T22:13:54,747 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/bd6592c37bb2:0-6 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-11-24T22:13:54,747 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/bd6592c37bb2:0-7 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-11-24T22:13:54,748 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK f0252adfc27b54a76a05585f3f45bf90, store=A 2024-11-24T22:13:54,748 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/bd6592c37bb2:0-5 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1732486433576, max timestamp=1732486434743Num uniques 859; 2024-11-24T22:13:54,748 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-24T22:13:54,748 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/bd6592c37bb2:0-5 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-11-24T22:13:54,748 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK f0252adfc27b54a76a05585f3f45bf90, store=B 2024-11-24T22:13:54,748 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-24T22:13:54,748 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK f0252adfc27b54a76a05585f3f45bf90, store=C 2024-11-24T22:13:54,748 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-24T22:13:54,758 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/f0252adfc27b54a76a05585f3f45bf90/.tmp/A/36b23a52fa034f5c9277bf34526f85c8 is 50, key is test_row_0/A:col10/1732486434743/Put/seqid=0 2024-11-24T22:13:54,774 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45929 is added to blk_1073741861_1037 (size=12151) 2024-11-24T22:13:54,775 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=8.95 KB at sequenceid=193 (bloomFilter=true), to=hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/f0252adfc27b54a76a05585f3f45bf90/.tmp/A/36b23a52fa034f5c9277bf34526f85c8 2024-11-24T22:13:54,786 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/bd6592c37bb2:0-2 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=209, cellSize=9565, totalHeapSize=21976, min timestamp=1732486434764, max timestamp=1732486434783Num uniques 859; 2024-11-24T22:13:54,786 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/bd6592c37bb2:0-8 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=209, cellSize=9565, totalHeapSize=21976, min timestamp=1732486434764, max timestamp=1732486434783Num uniques 859; 2024-11-24T22:13:54,786 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/bd6592c37bb2:0-0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=209, cellSize=9565, totalHeapSize=21976, min timestamp=1732486434764, max timestamp=1732486434783Num uniques 859; 2024-11-24T22:13:54,786 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/bd6592c37bb2:0-2 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=9565, suffix heap size=117064,new segment heap size=21976 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=209 2024-11-24T22:13:54,786 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/bd6592c37bb2:0-8 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=9565, suffix heap size=117064,new segment heap size=21976 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=209 2024-11-24T22:13:54,786 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/bd6592c37bb2:0-0 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=9565, suffix heap size=117064,new segment heap size=21976 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=209 2024-11-24T22:13:54,790 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/f0252adfc27b54a76a05585f3f45bf90/.tmp/B/f2c3972b4102489780168f825b2d9169 is 50, key is test_row_0/B:col10/1732486434743/Put/seqid=0 2024-11-24T22:13:54,828 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/bd6592c37bb2:0-9 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=418, cellSize=19138, totalHeapSize=43712, min timestamp=1732486434787, max timestamp=1732486434822Num uniques 1068; 2024-11-24T22:13:54,828 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/bd6592c37bb2:0-3 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=418, cellSize=19138, totalHeapSize=43712, min timestamp=1732486434787, max timestamp=1732486434822Num uniques 1068; 2024-11-24T22:13:54,828 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/bd6592c37bb2:0-9 {}] regionserver.CompactionPipeline(163): Suffix data size=48908, new segment data size=19138, suffix heap size=139040,new segment heap size=43712 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1068, new segment cells count=418 2024-11-24T22:13:54,828 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/bd6592c37bb2:0-3 {}] regionserver.CompactionPipeline(163): Suffix data size=48908, new segment data size=19138, suffix heap size=139040,new segment heap size=43712 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1068, new segment cells count=418 2024-11-24T22:13:54,830 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/bd6592c37bb2:0-1 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=168, cellSize=7688, totalHeapSize=17712, min timestamp=1732486434787, max timestamp=1732486434822Num uniques 1068; 2024-11-24T22:13:54,831 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45929 is added to blk_1073741862_1038 (size=12151) 2024-11-24T22:13:54,831 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/bd6592c37bb2:0-1 {}] regionserver.CompactionPipeline(163): Suffix data size=48908, new segment data size=7688, suffix heap size=139040,new segment heap size=17712 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1068, new segment cells count=168 2024-11-24T22:13:54,833 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=8.95 KB at sequenceid=193 (bloomFilter=true), to=hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/f0252adfc27b54a76a05585f3f45bf90/.tmp/B/f2c3972b4102489780168f825b2d9169 2024-11-24T22:13:54,849 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/f0252adfc27b54a76a05585f3f45bf90/.tmp/C/de4b0d358bae4edb83ce2b00b511adb1 is 50, key is test_row_0/C:col10/1732486434743/Put/seqid=0 2024-11-24T22:13:54,865 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45929 is added to blk_1073741863_1039 (size=12151) 2024-11-24T22:13:54,867 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40087 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0252adfc27b54a76a05585f3f45bf90, server=bd6592c37bb2,40087,1732486423713 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-24T22:13:54,867 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40087 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0252adfc27b54a76a05585f3f45bf90, server=bd6592c37bb2,40087,1732486423713 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-24T22:13:54,868 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40087 {}] ipc.CallRunner(138): callId: 41 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36548 deadline: 1732486494856, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0252adfc27b54a76a05585f3f45bf90, server=bd6592c37bb2,40087,1732486423713 2024-11-24T22:13:54,869 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40087 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0252adfc27b54a76a05585f3f45bf90, server=bd6592c37bb2,40087,1732486423713 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-24T22:13:54,869 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/bd6592c37bb2:0-4 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=277, cellSize=12682, totalHeapSize=29048, min timestamp=1732486434852, max timestamp=1732486434864Num uniques 1277; 2024-11-24T22:13:54,870 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/bd6592c37bb2:0-7 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=277, cellSize=12682, totalHeapSize=29048, min timestamp=1732486434852, max timestamp=1732486434864Num uniques 1027; 2024-11-24T22:13:54,870 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/bd6592c37bb2:0-7 {}] regionserver.CompactionPipeline(163): Suffix data size=47032, new segment data size=12682, suffix heap size=134776,new segment heap size=29048 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1027, new segment cells count=277 2024-11-24T22:13:54,870 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732486426439.f0252adfc27b54a76a05585f3f45bf90., hostname=bd6592c37bb2,40087,1732486423713, seqNum=2 , the old value is region=TestAcidGuarantees,,1732486426439.f0252adfc27b54a76a05585f3f45bf90., hostname=bd6592c37bb2,40087,1732486423713, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0252adfc27b54a76a05585f3f45bf90, server=bd6592c37bb2,40087,1732486423713 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T22:13:54,870 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732486426439.f0252adfc27b54a76a05585f3f45bf90., hostname=bd6592c37bb2,40087,1732486423713, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0252adfc27b54a76a05585f3f45bf90, server=bd6592c37bb2,40087,1732486423713 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T22:13:54,870 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732486426439.f0252adfc27b54a76a05585f3f45bf90., hostname=bd6592c37bb2,40087,1732486423713, seqNum=2 because the exception is null or not the one we care about 2024-11-24T22:13:54,870 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/bd6592c37bb2:0-4 {}] regionserver.CompactionPipeline(163): Suffix data size=58482, new segment data size=12682, suffix heap size=160776,new segment heap size=29048 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1277, new segment cells count=277 2024-11-24T22:13:54,870 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/bd6592c37bb2:0-6 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=277, cellSize=12682, totalHeapSize=29048, min timestamp=1732486434852, max timestamp=1732486434864Num uniques 1277; 2024-11-24T22:13:54,872 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40087 {}] ipc.CallRunner(138): callId: 72 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36562 deadline: 1732486494855, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0252adfc27b54a76a05585f3f45bf90, server=bd6592c37bb2,40087,1732486423713 2024-11-24T22:13:54,872 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40087 {}] ipc.CallRunner(138): callId: 57 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36590 deadline: 1732486494861, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0252adfc27b54a76a05585f3f45bf90, server=bd6592c37bb2,40087,1732486423713 2024-11-24T22:13:54,872 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/bd6592c37bb2:0-6 {}] regionserver.CompactionPipeline(163): Suffix data size=58482, new segment data size=12682, suffix heap size=160776,new segment heap size=29048 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1277, new segment cells count=277 2024-11-24T22:13:54,872 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732486426439.f0252adfc27b54a76a05585f3f45bf90., hostname=bd6592c37bb2,40087,1732486423713, seqNum=2 , the old value is region=TestAcidGuarantees,,1732486426439.f0252adfc27b54a76a05585f3f45bf90., hostname=bd6592c37bb2,40087,1732486423713, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0252adfc27b54a76a05585f3f45bf90, server=bd6592c37bb2,40087,1732486423713 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T22:13:54,872 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732486426439.f0252adfc27b54a76a05585f3f45bf90., hostname=bd6592c37bb2,40087,1732486423713, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0252adfc27b54a76a05585f3f45bf90, server=bd6592c37bb2,40087,1732486423713 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T22:13:54,872 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732486426439.f0252adfc27b54a76a05585f3f45bf90., hostname=bd6592c37bb2,40087,1732486423713, seqNum=2 because the exception is null or not the one we care about 2024-11-24T22:13:54,873 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732486426439.f0252adfc27b54a76a05585f3f45bf90., hostname=bd6592c37bb2,40087,1732486423713, seqNum=2 , the old value is region=TestAcidGuarantees,,1732486426439.f0252adfc27b54a76a05585f3f45bf90., hostname=bd6592c37bb2,40087,1732486423713, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0252adfc27b54a76a05585f3f45bf90, server=bd6592c37bb2,40087,1732486423713 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T22:13:54,873 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732486426439.f0252adfc27b54a76a05585f3f45bf90., hostname=bd6592c37bb2,40087,1732486423713, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0252adfc27b54a76a05585f3f45bf90, server=bd6592c37bb2,40087,1732486423713 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T22:13:54,873 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732486426439.f0252adfc27b54a76a05585f3f45bf90., hostname=bd6592c37bb2,40087,1732486423713, seqNum=2 because the exception is null or not the one we care about 2024-11-24T22:13:54,937 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/bd6592c37bb2:0-2 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=236, cellSize=10806, totalHeapSize=24784, min timestamp=1732486434905, max timestamp=1732486434932Num uniques 1136; 2024-11-24T22:13:54,937 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/bd6592c37bb2:0-5 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=236, cellSize=10806, totalHeapSize=24784, min timestamp=1732486434905, max timestamp=1732486434932Num uniques 1136; 2024-11-24T22:13:54,937 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/bd6592c37bb2:0-2 {}] regionserver.CompactionPipeline(163): Suffix data size=52026, new segment data size=10806, suffix heap size=146112,new segment heap size=24784 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1136, new segment cells count=236 2024-11-24T22:13:54,937 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/bd6592c37bb2:0-8 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=236, cellSize=10806, totalHeapSize=24784, min timestamp=1732486434905, max timestamp=1732486434932Num uniques 1136; 2024-11-24T22:13:54,938 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/bd6592c37bb2:0-8 {}] regionserver.CompactionPipeline(163): Suffix data size=52026, new segment data size=10806, suffix heap size=146112,new segment heap size=24784 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1136, new segment cells count=236 2024-11-24T22:13:54,938 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/bd6592c37bb2:0-5 {}] regionserver.CompactionPipeline(163): Suffix data size=52026, new segment data size=10806, suffix heap size=146112,new segment heap size=24784 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1136, new segment cells count=236 2024-11-24T22:13:54,995 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/bd6592c37bb2:0-9 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=295, cellSize=13510, totalHeapSize=30920, min timestamp=1732486434966, max timestamp=1732486434990Num uniques 1095; 2024-11-24T22:13:54,995 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/bd6592c37bb2:0-0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=295, cellSize=13510, totalHeapSize=30920, min timestamp=1732486434966, max timestamp=1732486434990Num uniques 1095; 2024-11-24T22:13:54,995 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/bd6592c37bb2:0-9 {}] regionserver.CompactionPipeline(163): Suffix data size=50150, new segment data size=13510, suffix heap size=141848,new segment heap size=30920 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1095, new segment cells count=295 2024-11-24T22:13:54,995 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/bd6592c37bb2:0-0 {}] regionserver.CompactionPipeline(163): Suffix data size=50150, new segment data size=13510, suffix heap size=141848,new segment heap size=30920 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1095, new segment cells count=295 2024-11-24T22:13:54,995 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/bd6592c37bb2:0-3 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=295, cellSize=13510, totalHeapSize=30920, min timestamp=1732486434966, max timestamp=1732486434990Num uniques 1095; 2024-11-24T22:13:54,995 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/bd6592c37bb2:0-3 {}] regionserver.CompactionPipeline(163): Suffix data size=50150, new segment data size=13510, suffix heap size=141848,new segment heap size=30920 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1095, new segment cells count=295 2024-11-24T22:13:55,024 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40087 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0252adfc27b54a76a05585f3f45bf90, server=bd6592c37bb2,40087,1732486423713 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-24T22:13:55,027 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40087 {}] ipc.CallRunner(138): callId: 46 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36548 deadline: 1732486495015, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0252adfc27b54a76a05585f3f45bf90, server=bd6592c37bb2,40087,1732486423713 2024-11-24T22:13:55,027 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40087 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0252adfc27b54a76a05585f3f45bf90, server=bd6592c37bb2,40087,1732486423713 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-24T22:13:55,027 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/bd6592c37bb2:0-7 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=304, cellSize=13920, totalHeapSize=31856, min timestamp=1732486435007, max timestamp=1732486435022Num uniques 1154; 2024-11-24T22:13:55,028 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40087 {}] ipc.CallRunner(138): callId: 62 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36590 deadline: 1732486495020, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0252adfc27b54a76a05585f3f45bf90, server=bd6592c37bb2,40087,1732486423713 2024-11-24T22:13:55,028 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/bd6592c37bb2:0-7 {}] regionserver.CompactionPipeline(163): Suffix data size=52850, new segment data size=13920, suffix heap size=147984,new segment heap size=31856 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1154, new segment cells count=304 2024-11-24T22:13:55,028 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732486426439.f0252adfc27b54a76a05585f3f45bf90., hostname=bd6592c37bb2,40087,1732486423713, seqNum=2 , the old value is region=TestAcidGuarantees,,1732486426439.f0252adfc27b54a76a05585f3f45bf90., hostname=bd6592c37bb2,40087,1732486423713, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0252adfc27b54a76a05585f3f45bf90, server=bd6592c37bb2,40087,1732486423713 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T22:13:55,028 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732486426439.f0252adfc27b54a76a05585f3f45bf90., hostname=bd6592c37bb2,40087,1732486423713, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0252adfc27b54a76a05585f3f45bf90, server=bd6592c37bb2,40087,1732486423713 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T22:13:55,028 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732486426439.f0252adfc27b54a76a05585f3f45bf90., hostname=bd6592c37bb2,40087,1732486423713, seqNum=2 because the exception is null or not the one we care about 2024-11-24T22:13:55,030 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/bd6592c37bb2:0-1 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=304, cellSize=13920, totalHeapSize=31856, min timestamp=1732486435007, max timestamp=1732486435022Num uniques 1154; 2024-11-24T22:13:55,030 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/bd6592c37bb2:0-1 {}] regionserver.CompactionPipeline(163): Suffix data size=52850, new segment data size=13920, suffix heap size=147984,new segment heap size=31856 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1154, new segment cells count=304 2024-11-24T22:13:55,030 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732486426439.f0252adfc27b54a76a05585f3f45bf90., hostname=bd6592c37bb2,40087,1732486423713, seqNum=2 , the old value is region=TestAcidGuarantees,,1732486426439.f0252adfc27b54a76a05585f3f45bf90., hostname=bd6592c37bb2,40087,1732486423713, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0252adfc27b54a76a05585f3f45bf90, server=bd6592c37bb2,40087,1732486423713 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T22:13:55,030 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732486426439.f0252adfc27b54a76a05585f3f45bf90., hostname=bd6592c37bb2,40087,1732486423713, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0252adfc27b54a76a05585f3f45bf90, server=bd6592c37bb2,40087,1732486423713 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T22:13:55,030 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732486426439.f0252adfc27b54a76a05585f3f45bf90., hostname=bd6592c37bb2,40087,1732486423713, seqNum=2 because the exception is null or not the one we care about 2024-11-24T22:13:55,030 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/bd6592c37bb2:0-4 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=304, cellSize=13920, totalHeapSize=31856, min timestamp=1732486435007, max timestamp=1732486435022Num uniques 1154; 2024-11-24T22:13:55,030 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/bd6592c37bb2:0-4 {}] regionserver.CompactionPipeline(163): Suffix data size=52850, new segment data size=13920, suffix heap size=147984,new segment heap size=31856 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1154, new segment cells count=304 2024-11-24T22:13:55,031 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37701 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=9 2024-11-24T22:13:55,032 INFO [RPCClient-NioEventLoopGroup-4-8 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-24T22:13:55,032 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37701 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-24T22:13:55,034 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37701 {}] procedure2.ProcedureExecutor(1139): Stored pid=11, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=11, table=TestAcidGuarantees 2024-11-24T22:13:55,035 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37701 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=11 2024-11-24T22:13:55,035 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=11, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=11, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-24T22:13:55,036 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=11, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=11, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-24T22:13:55,037 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=12, ppid=11, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-24T22:13:55,082 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40087 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0252adfc27b54a76a05585f3f45bf90, server=bd6592c37bb2,40087,1732486423713 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-24T22:13:55,082 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40087 {}] ipc.CallRunner(138): callId: 83 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36562 deadline: 1732486495074, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0252adfc27b54a76a05585f3f45bf90, server=bd6592c37bb2,40087,1732486423713 2024-11-24T22:13:55,085 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732486426439.f0252adfc27b54a76a05585f3f45bf90., hostname=bd6592c37bb2,40087,1732486423713, seqNum=2 , the old value is region=TestAcidGuarantees,,1732486426439.f0252adfc27b54a76a05585f3f45bf90., hostname=bd6592c37bb2,40087,1732486423713, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0252adfc27b54a76a05585f3f45bf90, server=bd6592c37bb2,40087,1732486423713 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T22:13:55,085 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732486426439.f0252adfc27b54a76a05585f3f45bf90., hostname=bd6592c37bb2,40087,1732486423713, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0252adfc27b54a76a05585f3f45bf90, server=bd6592c37bb2,40087,1732486423713 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T22:13:55,085 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732486426439.f0252adfc27b54a76a05585f3f45bf90., hostname=bd6592c37bb2,40087,1732486423713, seqNum=2 because the exception is null or not the one we care about 2024-11-24T22:13:55,085 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/bd6592c37bb2:0-6 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=313, cellSize=14328, totalHeapSize=32792, min timestamp=1732486435053, max timestamp=1732486435080Num uniques 1163; 2024-11-24T22:13:55,086 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/bd6592c37bb2:0-6 {}] regionserver.CompactionPipeline(163): Suffix data size=53258, new segment data size=14328, suffix heap size=148920,new segment heap size=32792 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1163, new segment cells count=313 2024-11-24T22:13:55,086 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/bd6592c37bb2:0-8 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=313, cellSize=14328, totalHeapSize=32792, min timestamp=1732486435053, max timestamp=1732486435080Num uniques 1163; 2024-11-24T22:13:55,086 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/bd6592c37bb2:0-2 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=313, cellSize=14328, totalHeapSize=32792, min timestamp=1732486435053, max timestamp=1732486435080Num uniques 1163; 2024-11-24T22:13:55,086 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40087 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0252adfc27b54a76a05585f3f45bf90, server=bd6592c37bb2,40087,1732486423713 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-24T22:13:55,086 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/bd6592c37bb2:0-8 {}] regionserver.CompactionPipeline(163): Suffix data size=53258, new segment data size=14328, suffix heap size=148920,new segment heap size=32792 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1163, new segment cells count=313 2024-11-24T22:13:55,086 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40087 {}] ipc.CallRunner(138): callId: 96 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36532 deadline: 1732486495081, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0252adfc27b54a76a05585f3f45bf90, server=bd6592c37bb2,40087,1732486423713 2024-11-24T22:13:55,086 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732486426439.f0252adfc27b54a76a05585f3f45bf90., hostname=bd6592c37bb2,40087,1732486423713, seqNum=2 , the old value is region=TestAcidGuarantees,,1732486426439.f0252adfc27b54a76a05585f3f45bf90., hostname=bd6592c37bb2,40087,1732486423713, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0252adfc27b54a76a05585f3f45bf90, server=bd6592c37bb2,40087,1732486423713 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T22:13:55,086 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732486426439.f0252adfc27b54a76a05585f3f45bf90., hostname=bd6592c37bb2,40087,1732486423713, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0252adfc27b54a76a05585f3f45bf90, server=bd6592c37bb2,40087,1732486423713 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T22:13:55,087 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732486426439.f0252adfc27b54a76a05585f3f45bf90., hostname=bd6592c37bb2,40087,1732486423713, seqNum=2 because the exception is null or not the one we care about 2024-11-24T22:13:55,086 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/bd6592c37bb2:0-2 {}] regionserver.CompactionPipeline(163): Suffix data size=53258, new segment data size=14328, suffix heap size=148920,new segment heap size=32792 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1163, new segment cells count=313 2024-11-24T22:13:55,141 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37701 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=11 2024-11-24T22:13:55,160 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/bd6592c37bb2:0-9 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=322, cellSize=14742, totalHeapSize=33728, min timestamp=1732486435117, max timestamp=1732486435155Num uniques 1172; 2024-11-24T22:13:55,160 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/bd6592c37bb2:0-9 {}] regionserver.CompactionPipeline(163): Suffix data size=53672, new segment data size=14742, suffix heap size=149856,new segment heap size=33728 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1172, new segment cells count=322 2024-11-24T22:13:55,160 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/bd6592c37bb2:0-5 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=322, cellSize=14742, totalHeapSize=33728, min timestamp=1732486435117, max timestamp=1732486435155Num uniques 1172; 2024-11-24T22:13:55,160 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/bd6592c37bb2:0-0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=322, cellSize=14742, totalHeapSize=33728, min timestamp=1732486435117, max timestamp=1732486435155Num uniques 1172; 2024-11-24T22:13:55,160 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/bd6592c37bb2:0-5 {}] regionserver.CompactionPipeline(163): Suffix data size=53672, new segment data size=14742, suffix heap size=149856,new segment heap size=33728 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1172, new segment cells count=322 2024-11-24T22:13:55,161 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/bd6592c37bb2:0-0 {}] regionserver.CompactionPipeline(163): Suffix data size=53672, new segment data size=14742, suffix heap size=149856,new segment heap size=33728 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1172, new segment cells count=322 2024-11-24T22:13:55,190 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=40087 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=12 2024-11-24T22:13:55,190 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bd6592c37bb2:0-0 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732486426439.f0252adfc27b54a76a05585f3f45bf90. 2024-11-24T22:13:55,190 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bd6592c37bb2:0-0 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732486426439.f0252adfc27b54a76a05585f3f45bf90. as already flushing 2024-11-24T22:13:55,190 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bd6592c37bb2:0-0 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732486426439.f0252adfc27b54a76a05585f3f45bf90. 2024-11-24T22:13:55,190 ERROR [RS_FLUSH_OPERATIONS-regionserver/bd6592c37bb2:0-0 {event_type=RS_FLUSH_REGIONS, pid=12}] handler.RSProcedureHandler(58): pid=12 java.io.IOException: Unable to complete flush {ENCODED => f0252adfc27b54a76a05585f3f45bf90, NAME => 'TestAcidGuarantees,,1732486426439.f0252adfc27b54a76a05585f3f45bf90.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T22:13:55,190 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bd6592c37bb2:0-0 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=12 java.io.IOException: Unable to complete flush {ENCODED => f0252adfc27b54a76a05585f3f45bf90, NAME => 'TestAcidGuarantees,,1732486426439.f0252adfc27b54a76a05585f3f45bf90.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T22:13:55,191 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37701 {}] master.HMaster(4177): Remote procedure failed, pid=12 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => f0252adfc27b54a76a05585f3f45bf90, NAME => 'TestAcidGuarantees,,1732486426439.f0252adfc27b54a76a05585f3f45bf90.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => f0252adfc27b54a76a05585f3f45bf90, NAME => 'TestAcidGuarantees,,1732486426439.f0252adfc27b54a76a05585f3f45bf90.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T22:13:55,207 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40087 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0252adfc27b54a76a05585f3f45bf90, server=bd6592c37bb2,40087,1732486423713 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-24T22:13:55,208 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40087 {}] ipc.CallRunner(138): callId: 71 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36590 deadline: 1732486495199, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0252adfc27b54a76a05585f3f45bf90, server=bd6592c37bb2,40087,1732486423713 2024-11-24T22:13:55,208 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40087 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0252adfc27b54a76a05585f3f45bf90, server=bd6592c37bb2,40087,1732486423713 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-24T22:13:55,208 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40087 {}] ipc.CallRunner(138): callId: 85 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36562 deadline: 1732486495199, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0252adfc27b54a76a05585f3f45bf90, server=bd6592c37bb2,40087,1732486423713 2024-11-24T22:13:55,209 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40087 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0252adfc27b54a76a05585f3f45bf90, server=bd6592c37bb2,40087,1732486423713 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-24T22:13:55,209 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40087 {}] ipc.CallRunner(138): callId: 98 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36532 deadline: 1732486495200, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0252adfc27b54a76a05585f3f45bf90, server=bd6592c37bb2,40087,1732486423713 2024-11-24T22:13:55,211 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/bd6592c37bb2:0-3 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=231, cellSize=10576, totalHeapSize=24264, min timestamp=1732486435186, max timestamp=1732486435205Num uniques 1181; 2024-11-24T22:13:55,211 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/bd6592c37bb2:0-3 {}] regionserver.CompactionPipeline(163): Suffix data size=54086, new segment data size=10576, suffix heap size=150792,new segment heap size=24264 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1181, new segment cells count=231 2024-11-24T22:13:55,211 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/bd6592c37bb2:0-1 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=231, cellSize=10576, totalHeapSize=24264, min timestamp=1732486435186, max timestamp=1732486435205Num uniques 1181; 2024-11-24T22:13:55,211 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/bd6592c37bb2:0-7 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=231, cellSize=10576, totalHeapSize=24264, min timestamp=1732486435186, max timestamp=1732486435205Num uniques 1181; 2024-11-24T22:13:55,211 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/bd6592c37bb2:0-1 {}] regionserver.CompactionPipeline(163): Suffix data size=54086, new segment data size=10576, suffix heap size=150792,new segment heap size=24264 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1181, new segment cells count=231 2024-11-24T22:13:55,212 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732486426439.f0252adfc27b54a76a05585f3f45bf90., hostname=bd6592c37bb2,40087,1732486423713, seqNum=2 , the old value is region=TestAcidGuarantees,,1732486426439.f0252adfc27b54a76a05585f3f45bf90., hostname=bd6592c37bb2,40087,1732486423713, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0252adfc27b54a76a05585f3f45bf90, server=bd6592c37bb2,40087,1732486423713 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T22:13:55,212 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732486426439.f0252adfc27b54a76a05585f3f45bf90., hostname=bd6592c37bb2,40087,1732486423713, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0252adfc27b54a76a05585f3f45bf90, server=bd6592c37bb2,40087,1732486423713 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T22:13:55,212 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732486426439.f0252adfc27b54a76a05585f3f45bf90., hostname=bd6592c37bb2,40087,1732486423713, seqNum=2 because the exception is null or not the one we care about 2024-11-24T22:13:55,212 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732486426439.f0252adfc27b54a76a05585f3f45bf90., hostname=bd6592c37bb2,40087,1732486423713, seqNum=2 , the old value is region=TestAcidGuarantees,,1732486426439.f0252adfc27b54a76a05585f3f45bf90., hostname=bd6592c37bb2,40087,1732486423713, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0252adfc27b54a76a05585f3f45bf90, server=bd6592c37bb2,40087,1732486423713 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T22:13:55,212 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732486426439.f0252adfc27b54a76a05585f3f45bf90., hostname=bd6592c37bb2,40087,1732486423713, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0252adfc27b54a76a05585f3f45bf90, server=bd6592c37bb2,40087,1732486423713 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T22:13:55,212 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732486426439.f0252adfc27b54a76a05585f3f45bf90., hostname=bd6592c37bb2,40087,1732486423713, seqNum=2 , the old value is region=TestAcidGuarantees,,1732486426439.f0252adfc27b54a76a05585f3f45bf90., hostname=bd6592c37bb2,40087,1732486423713, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0252adfc27b54a76a05585f3f45bf90, server=bd6592c37bb2,40087,1732486423713 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T22:13:55,213 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732486426439.f0252adfc27b54a76a05585f3f45bf90., hostname=bd6592c37bb2,40087,1732486423713, seqNum=2 because the exception is null or not the one we care about 2024-11-24T22:13:55,213 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732486426439.f0252adfc27b54a76a05585f3f45bf90., hostname=bd6592c37bb2,40087,1732486423713, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0252adfc27b54a76a05585f3f45bf90, server=bd6592c37bb2,40087,1732486423713 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T22:13:55,213 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732486426439.f0252adfc27b54a76a05585f3f45bf90., hostname=bd6592c37bb2,40087,1732486423713, seqNum=2 because the exception is null or not the one we care about 2024-11-24T22:13:55,213 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/bd6592c37bb2:0-7 {}] regionserver.CompactionPipeline(163): Suffix data size=54086, new segment data size=10576, suffix heap size=150792,new segment heap size=24264 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1181, new segment cells count=231 2024-11-24T22:13:55,267 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=8.95 KB at sequenceid=193 (bloomFilter=true), to=hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/f0252adfc27b54a76a05585f3f45bf90/.tmp/C/de4b0d358bae4edb83ce2b00b511adb1 2024-11-24T22:13:55,277 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/bd6592c37bb2:0-4 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=240, cellSize=10990, totalHeapSize=25200, min timestamp=1732486435239, max timestamp=1732486435267Num uniques 1090; 2024-11-24T22:13:55,277 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/bd6592c37bb2:0-8 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=240, cellSize=10990, totalHeapSize=25200, min timestamp=1732486435239, max timestamp=1732486435267Num uniques 1090; 2024-11-24T22:13:55,277 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/bd6592c37bb2:0-4 {}] regionserver.CompactionPipeline(163): Suffix data size=49920, new segment data size=10990, suffix heap size=141328,new segment heap size=25200 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1090, new segment cells count=240 2024-11-24T22:13:55,277 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/bd6592c37bb2:0-8 {}] regionserver.CompactionPipeline(163): Suffix data size=49920, new segment data size=10990, suffix heap size=141328,new segment heap size=25200 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1090, new segment cells count=240 2024-11-24T22:13:55,277 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/bd6592c37bb2:0-6 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=240, cellSize=10990, totalHeapSize=25200, min timestamp=1732486435239, max timestamp=1732486435267Num uniques 1090; 2024-11-24T22:13:55,277 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/bd6592c37bb2:0-6 {}] regionserver.CompactionPipeline(163): Suffix data size=49920, new segment data size=10990, suffix heap size=141328,new segment heap size=25200 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1090, new segment cells count=240 2024-11-24T22:13:55,285 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/f0252adfc27b54a76a05585f3f45bf90/.tmp/A/36b23a52fa034f5c9277bf34526f85c8 as hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/f0252adfc27b54a76a05585f3f45bf90/A/36b23a52fa034f5c9277bf34526f85c8 2024-11-24T22:13:55,294 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/f0252adfc27b54a76a05585f3f45bf90/A/36b23a52fa034f5c9277bf34526f85c8, entries=150, sequenceid=193, filesize=11.9 K 2024-11-24T22:13:55,297 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/f0252adfc27b54a76a05585f3f45bf90/.tmp/B/f2c3972b4102489780168f825b2d9169 as hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/f0252adfc27b54a76a05585f3f45bf90/B/f2c3972b4102489780168f825b2d9169 2024-11-24T22:13:55,304 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/f0252adfc27b54a76a05585f3f45bf90/B/f2c3972b4102489780168f825b2d9169, entries=150, sequenceid=193, filesize=11.9 K 2024-11-24T22:13:55,305 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/f0252adfc27b54a76a05585f3f45bf90/.tmp/C/de4b0d358bae4edb83ce2b00b511adb1 as hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/f0252adfc27b54a76a05585f3f45bf90/C/de4b0d358bae4edb83ce2b00b511adb1 2024-11-24T22:13:55,318 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/f0252adfc27b54a76a05585f3f45bf90/C/de4b0d358bae4edb83ce2b00b511adb1, entries=150, sequenceid=193, filesize=11.9 K 2024-11-24T22:13:55,319 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~26.84 KB/27480, heapSize ~66.19 KB/67776, currentSize=93.93 KB/96180 for f0252adfc27b54a76a05585f3f45bf90 in 572ms, sequenceid=193, compaction requested=true 2024-11-24T22:13:55,320 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for f0252adfc27b54a76a05585f3f45bf90: 2024-11-24T22:13:55,320 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store f0252adfc27b54a76a05585f3f45bf90:A, priority=-2147483648, current under compaction store size is 1 2024-11-24T22:13:55,320 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-24T22:13:55,320 DEBUG [RS:0;bd6592c37bb2:40087-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-24T22:13:55,320 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store f0252adfc27b54a76a05585f3f45bf90:B, priority=-2147483648, current under compaction store size is 2 2024-11-24T22:13:55,320 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-24T22:13:55,320 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store f0252adfc27b54a76a05585f3f45bf90:C, priority=-2147483648, current under compaction store size is 3 2024-11-24T22:13:55,320 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-24T22:13:55,320 DEBUG [RS:0;bd6592c37bb2:40087-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-24T22:13:55,323 DEBUG [RS:0;bd6592c37bb2:40087-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36659 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-24T22:13:55,323 DEBUG [RS:0;bd6592c37bb2:40087-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36659 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-24T22:13:55,323 DEBUG [RS:0;bd6592c37bb2:40087-longCompactions-0 {}] regionserver.HStore(1541): f0252adfc27b54a76a05585f3f45bf90/B is initiating minor compaction (all files) 2024-11-24T22:13:55,323 DEBUG [RS:0;bd6592c37bb2:40087-shortCompactions-0 {}] regionserver.HStore(1541): f0252adfc27b54a76a05585f3f45bf90/A is initiating minor compaction (all files) 2024-11-24T22:13:55,323 INFO [RS:0;bd6592c37bb2:40087-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of f0252adfc27b54a76a05585f3f45bf90/B in TestAcidGuarantees,,1732486426439.f0252adfc27b54a76a05585f3f45bf90. 2024-11-24T22:13:55,323 INFO [RS:0;bd6592c37bb2:40087-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of f0252adfc27b54a76a05585f3f45bf90/A in TestAcidGuarantees,,1732486426439.f0252adfc27b54a76a05585f3f45bf90. 2024-11-24T22:13:55,323 INFO [RS:0;bd6592c37bb2:40087-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/f0252adfc27b54a76a05585f3f45bf90/B/be5ac8db548f4f02af9d4afa54364feb, hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/f0252adfc27b54a76a05585f3f45bf90/B/0f8a8c8188ae448487505488e234be20, hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/f0252adfc27b54a76a05585f3f45bf90/B/f2c3972b4102489780168f825b2d9169] into tmpdir=hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/f0252adfc27b54a76a05585f3f45bf90/.tmp, totalSize=35.8 K 2024-11-24T22:13:55,323 INFO [RS:0;bd6592c37bb2:40087-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/f0252adfc27b54a76a05585f3f45bf90/A/c8cf77fc903c408d900988a3a2b6a93c, hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/f0252adfc27b54a76a05585f3f45bf90/A/b146b26e6c524e5ca691fb2961fc5ffc, hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/f0252adfc27b54a76a05585f3f45bf90/A/36b23a52fa034f5c9277bf34526f85c8] into tmpdir=hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/f0252adfc27b54a76a05585f3f45bf90/.tmp, totalSize=35.8 K 2024-11-24T22:13:55,324 DEBUG [RS:0;bd6592c37bb2:40087-longCompactions-0 {}] compactions.Compactor(225): Compacting be5ac8db548f4f02af9d4afa54364feb, keycount=150, bloomtype=ROW, size=12.1 K, encoding=NONE, compression=NONE, seqNum=157, earliestPutTs=1732486431186 2024-11-24T22:13:55,324 DEBUG [RS:0;bd6592c37bb2:40087-shortCompactions-0 {}] compactions.Compactor(225): Compacting c8cf77fc903c408d900988a3a2b6a93c, keycount=150, bloomtype=ROW, size=12.1 K, encoding=NONE, compression=NONE, seqNum=157, earliestPutTs=1732486431186 2024-11-24T22:13:55,325 DEBUG [RS:0;bd6592c37bb2:40087-longCompactions-0 {}] compactions.Compactor(225): Compacting 0f8a8c8188ae448487505488e234be20, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=169, earliestPutTs=1732486432365 2024-11-24T22:13:55,325 DEBUG [RS:0;bd6592c37bb2:40087-shortCompactions-0 {}] compactions.Compactor(225): Compacting b146b26e6c524e5ca691fb2961fc5ffc, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=169, earliestPutTs=1732486432365 2024-11-24T22:13:55,326 DEBUG [RS:0;bd6592c37bb2:40087-shortCompactions-0 {}] compactions.Compactor(225): Compacting 36b23a52fa034f5c9277bf34526f85c8, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=193, earliestPutTs=1732486433576 2024-11-24T22:13:55,326 DEBUG [RS:0;bd6592c37bb2:40087-longCompactions-0 {}] compactions.Compactor(225): Compacting f2c3972b4102489780168f825b2d9169, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=193, earliestPutTs=1732486433576 2024-11-24T22:13:55,332 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40087 {}] regionserver.HRegion(8855): Flush requested on f0252adfc27b54a76a05585f3f45bf90 2024-11-24T22:13:55,332 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing f0252adfc27b54a76a05585f3f45bf90 3/3 column families, dataSize=114.05 KB heapSize=317.63 KB 2024-11-24T22:13:55,337 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK f0252adfc27b54a76a05585f3f45bf90, store=A 2024-11-24T22:13:55,337 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-24T22:13:55,337 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK f0252adfc27b54a76a05585f3f45bf90, store=B 2024-11-24T22:13:55,337 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-24T22:13:55,337 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK f0252adfc27b54a76a05585f3f45bf90, store=C 2024-11-24T22:13:55,337 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-24T22:13:55,344 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=40087 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=12 2024-11-24T22:13:55,344 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bd6592c37bb2:0-1 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732486426439.f0252adfc27b54a76a05585f3f45bf90. 2024-11-24T22:13:55,344 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bd6592c37bb2:0-1 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732486426439.f0252adfc27b54a76a05585f3f45bf90. as already flushing 2024-11-24T22:13:55,345 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bd6592c37bb2:0-1 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732486426439.f0252adfc27b54a76a05585f3f45bf90. 2024-11-24T22:13:55,345 ERROR [RS_FLUSH_OPERATIONS-regionserver/bd6592c37bb2:0-1 {event_type=RS_FLUSH_REGIONS, pid=12}] handler.RSProcedureHandler(58): pid=12 java.io.IOException: Unable to complete flush {ENCODED => f0252adfc27b54a76a05585f3f45bf90, NAME => 'TestAcidGuarantees,,1732486426439.f0252adfc27b54a76a05585f3f45bf90.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T22:13:55,345 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bd6592c37bb2:0-1 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=12 java.io.IOException: Unable to complete flush {ENCODED => f0252adfc27b54a76a05585f3f45bf90, NAME => 'TestAcidGuarantees,,1732486426439.f0252adfc27b54a76a05585f3f45bf90.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T22:13:55,345 INFO [RS:0;bd6592c37bb2:40087-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): f0252adfc27b54a76a05585f3f45bf90#B#compaction#27 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-24T22:13:55,346 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37701 {}] master.HMaster(4177): Remote procedure failed, pid=12 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => f0252adfc27b54a76a05585f3f45bf90, NAME => 'TestAcidGuarantees,,1732486426439.f0252adfc27b54a76a05585f3f45bf90.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => f0252adfc27b54a76a05585f3f45bf90, NAME => 'TestAcidGuarantees,,1732486426439.f0252adfc27b54a76a05585f3f45bf90.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T22:13:55,347 INFO [RS:0;bd6592c37bb2:40087-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): f0252adfc27b54a76a05585f3f45bf90#A#compaction#28 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-24T22:13:55,348 DEBUG [RS:0;bd6592c37bb2:40087-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/f0252adfc27b54a76a05585f3f45bf90/.tmp/A/f3da0b40cc124272a32346cc906d98c5 is 50, key is test_row_0/A:col10/1732486434743/Put/seqid=0 2024-11-24T22:13:55,351 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37701 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=11 2024-11-24T22:13:55,347 DEBUG [RS:0;bd6592c37bb2:40087-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/f0252adfc27b54a76a05585f3f45bf90/.tmp/B/ffc9abede25d4d48a0ffc2479587f52a is 50, key is test_row_0/B:col10/1732486434743/Put/seqid=0 2024-11-24T22:13:55,356 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/f0252adfc27b54a76a05585f3f45bf90/.tmp/A/999745bc0c3c48e495f7184a9cd014a8 is 50, key is test_row_0/A:col10/1732486435332/Put/seqid=0 2024-11-24T22:13:55,361 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40087 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0252adfc27b54a76a05585f3f45bf90, server=bd6592c37bb2,40087,1732486423713 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-24T22:13:55,361 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40087 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0252adfc27b54a76a05585f3f45bf90, server=bd6592c37bb2,40087,1732486423713 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-24T22:13:55,361 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40087 {}] ipc.CallRunner(138): callId: 74 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36590 deadline: 1732486495352, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0252adfc27b54a76a05585f3f45bf90, server=bd6592c37bb2,40087,1732486423713 2024-11-24T22:13:55,361 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40087 {}] ipc.CallRunner(138): callId: 101 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36532 deadline: 1732486495348, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0252adfc27b54a76a05585f3f45bf90, server=bd6592c37bb2,40087,1732486423713 2024-11-24T22:13:55,362 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732486426439.f0252adfc27b54a76a05585f3f45bf90., hostname=bd6592c37bb2,40087,1732486423713, seqNum=2 , the old value is region=TestAcidGuarantees,,1732486426439.f0252adfc27b54a76a05585f3f45bf90., hostname=bd6592c37bb2,40087,1732486423713, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0252adfc27b54a76a05585f3f45bf90, server=bd6592c37bb2,40087,1732486423713 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T22:13:55,362 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732486426439.f0252adfc27b54a76a05585f3f45bf90., hostname=bd6592c37bb2,40087,1732486423713, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0252adfc27b54a76a05585f3f45bf90, server=bd6592c37bb2,40087,1732486423713 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T22:13:55,362 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732486426439.f0252adfc27b54a76a05585f3f45bf90., hostname=bd6592c37bb2,40087,1732486423713, seqNum=2 because the exception is null or not the one we care about 2024-11-24T22:13:55,362 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732486426439.f0252adfc27b54a76a05585f3f45bf90., hostname=bd6592c37bb2,40087,1732486423713, seqNum=2 , the old value is region=TestAcidGuarantees,,1732486426439.f0252adfc27b54a76a05585f3f45bf90., hostname=bd6592c37bb2,40087,1732486423713, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0252adfc27b54a76a05585f3f45bf90, server=bd6592c37bb2,40087,1732486423713 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T22:13:55,362 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732486426439.f0252adfc27b54a76a05585f3f45bf90., hostname=bd6592c37bb2,40087,1732486423713, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0252adfc27b54a76a05585f3f45bf90, server=bd6592c37bb2,40087,1732486423713 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T22:13:55,363 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732486426439.f0252adfc27b54a76a05585f3f45bf90., hostname=bd6592c37bb2,40087,1732486423713, seqNum=2 because the exception is null or not the one we care about 2024-11-24T22:13:55,373 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45929 is added to blk_1073741864_1040 (size=12459) 2024-11-24T22:13:55,373 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40087 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0252adfc27b54a76a05585f3f45bf90, server=bd6592c37bb2,40087,1732486423713 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-24T22:13:55,374 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40087 {}] ipc.CallRunner(138): callId: 132 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36518 deadline: 1732486495362, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0252adfc27b54a76a05585f3f45bf90, server=bd6592c37bb2,40087,1732486423713 2024-11-24T22:13:55,374 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732486426439.f0252adfc27b54a76a05585f3f45bf90., hostname=bd6592c37bb2,40087,1732486423713, seqNum=2 , the old value is region=TestAcidGuarantees,,1732486426439.f0252adfc27b54a76a05585f3f45bf90., hostname=bd6592c37bb2,40087,1732486423713, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0252adfc27b54a76a05585f3f45bf90, server=bd6592c37bb2,40087,1732486423713 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T22:13:55,374 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40087 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0252adfc27b54a76a05585f3f45bf90, server=bd6592c37bb2,40087,1732486423713 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-24T22:13:55,374 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732486426439.f0252adfc27b54a76a05585f3f45bf90., hostname=bd6592c37bb2,40087,1732486423713, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0252adfc27b54a76a05585f3f45bf90, server=bd6592c37bb2,40087,1732486423713 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T22:13:55,374 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732486426439.f0252adfc27b54a76a05585f3f45bf90., hostname=bd6592c37bb2,40087,1732486423713, seqNum=2 because the exception is null or not the one we care about 2024-11-24T22:13:55,375 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40087 {}] ipc.CallRunner(138): callId: 89 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36562 deadline: 1732486495362, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0252adfc27b54a76a05585f3f45bf90, server=bd6592c37bb2,40087,1732486423713 2024-11-24T22:13:55,375 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732486426439.f0252adfc27b54a76a05585f3f45bf90., hostname=bd6592c37bb2,40087,1732486423713, seqNum=2 , the old value is region=TestAcidGuarantees,,1732486426439.f0252adfc27b54a76a05585f3f45bf90., hostname=bd6592c37bb2,40087,1732486423713, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0252adfc27b54a76a05585f3f45bf90, server=bd6592c37bb2,40087,1732486423713 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T22:13:55,375 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40087 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0252adfc27b54a76a05585f3f45bf90, server=bd6592c37bb2,40087,1732486423713 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-24T22:13:55,375 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732486426439.f0252adfc27b54a76a05585f3f45bf90., hostname=bd6592c37bb2,40087,1732486423713, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0252adfc27b54a76a05585f3f45bf90, server=bd6592c37bb2,40087,1732486423713 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T22:13:55,375 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732486426439.f0252adfc27b54a76a05585f3f45bf90., hostname=bd6592c37bb2,40087,1732486423713, seqNum=2 because the exception is null or not the one we care about 2024-11-24T22:13:55,375 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40087 {}] ipc.CallRunner(138): callId: 69 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36548 deadline: 1732486495362, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0252adfc27b54a76a05585f3f45bf90, server=bd6592c37bb2,40087,1732486423713 2024-11-24T22:13:55,376 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732486426439.f0252adfc27b54a76a05585f3f45bf90., hostname=bd6592c37bb2,40087,1732486423713, seqNum=2 , the old value is region=TestAcidGuarantees,,1732486426439.f0252adfc27b54a76a05585f3f45bf90., hostname=bd6592c37bb2,40087,1732486423713, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0252adfc27b54a76a05585f3f45bf90, server=bd6592c37bb2,40087,1732486423713 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T22:13:55,376 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732486426439.f0252adfc27b54a76a05585f3f45bf90., hostname=bd6592c37bb2,40087,1732486423713, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0252adfc27b54a76a05585f3f45bf90, server=bd6592c37bb2,40087,1732486423713 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T22:13:55,376 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732486426439.f0252adfc27b54a76a05585f3f45bf90., hostname=bd6592c37bb2,40087,1732486423713, seqNum=2 because the exception is null or not the one we care about 2024-11-24T22:13:55,376 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45929 is added to blk_1073741865_1041 (size=12459) 2024-11-24T22:13:55,390 DEBUG [RS:0;bd6592c37bb2:40087-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/f0252adfc27b54a76a05585f3f45bf90/.tmp/B/ffc9abede25d4d48a0ffc2479587f52a as hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/f0252adfc27b54a76a05585f3f45bf90/B/ffc9abede25d4d48a0ffc2479587f52a 2024-11-24T22:13:55,399 INFO [RS:0;bd6592c37bb2:40087-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in f0252adfc27b54a76a05585f3f45bf90/B of f0252adfc27b54a76a05585f3f45bf90 into ffc9abede25d4d48a0ffc2479587f52a(size=12.2 K), total size for store is 12.2 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-24T22:13:55,399 DEBUG [RS:0;bd6592c37bb2:40087-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for f0252adfc27b54a76a05585f3f45bf90: 2024-11-24T22:13:55,399 INFO [RS:0;bd6592c37bb2:40087-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732486426439.f0252adfc27b54a76a05585f3f45bf90., storeName=f0252adfc27b54a76a05585f3f45bf90/B, priority=13, startTime=1732486435320; duration=0sec 2024-11-24T22:13:55,400 DEBUG [RS:0;bd6592c37bb2:40087-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-24T22:13:55,400 DEBUG [RS:0;bd6592c37bb2:40087-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: f0252adfc27b54a76a05585f3f45bf90:B 2024-11-24T22:13:55,400 DEBUG [RS:0;bd6592c37bb2:40087-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-24T22:13:55,401 DEBUG [RS:0;bd6592c37bb2:40087-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36659 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-24T22:13:55,401 DEBUG [RS:0;bd6592c37bb2:40087-longCompactions-0 {}] regionserver.HStore(1541): f0252adfc27b54a76a05585f3f45bf90/C is initiating minor compaction (all files) 2024-11-24T22:13:55,402 INFO [RS:0;bd6592c37bb2:40087-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of f0252adfc27b54a76a05585f3f45bf90/C in TestAcidGuarantees,,1732486426439.f0252adfc27b54a76a05585f3f45bf90. 2024-11-24T22:13:55,402 INFO [RS:0;bd6592c37bb2:40087-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/f0252adfc27b54a76a05585f3f45bf90/C/bc92118f30b44a4995b050c164edd61f, hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/f0252adfc27b54a76a05585f3f45bf90/C/7bbbf641f1b44d9195ae936a79cb2684, hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/f0252adfc27b54a76a05585f3f45bf90/C/de4b0d358bae4edb83ce2b00b511adb1] into tmpdir=hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/f0252adfc27b54a76a05585f3f45bf90/.tmp, totalSize=35.8 K 2024-11-24T22:13:55,402 DEBUG [RS:0;bd6592c37bb2:40087-longCompactions-0 {}] compactions.Compactor(225): Compacting bc92118f30b44a4995b050c164edd61f, keycount=150, bloomtype=ROW, size=12.1 K, encoding=NONE, compression=NONE, seqNum=157, earliestPutTs=1732486431186 2024-11-24T22:13:55,403 DEBUG [RS:0;bd6592c37bb2:40087-longCompactions-0 {}] compactions.Compactor(225): Compacting 7bbbf641f1b44d9195ae936a79cb2684, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=169, earliestPutTs=1732486432365 2024-11-24T22:13:55,404 DEBUG [RS:0;bd6592c37bb2:40087-longCompactions-0 {}] compactions.Compactor(225): Compacting de4b0d358bae4edb83ce2b00b511adb1, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=193, earliestPutTs=1732486433576 2024-11-24T22:13:55,406 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45929 is added to blk_1073741866_1042 (size=14741) 2024-11-24T22:13:55,409 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=42.49 KB at sequenceid=382 (bloomFilter=true), to=hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/f0252adfc27b54a76a05585f3f45bf90/.tmp/A/999745bc0c3c48e495f7184a9cd014a8 2024-11-24T22:13:55,422 INFO [RS:0;bd6592c37bb2:40087-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): f0252adfc27b54a76a05585f3f45bf90#C#compaction#31 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-24T22:13:55,422 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/f0252adfc27b54a76a05585f3f45bf90/.tmp/B/7fd5a8a6e6934156aebd2bd7ed3c6562 is 50, key is test_row_0/B:col10/1732486435332/Put/seqid=0 2024-11-24T22:13:55,422 DEBUG [RS:0;bd6592c37bb2:40087-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/f0252adfc27b54a76a05585f3f45bf90/.tmp/C/c4712c657f28446ea38507429ade8164 is 50, key is test_row_0/C:col10/1732486434743/Put/seqid=0 2024-11-24T22:13:55,444 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45929 is added to blk_1073741867_1043 (size=12301) 2024-11-24T22:13:55,450 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45929 is added to blk_1073741868_1044 (size=12459) 2024-11-24T22:13:55,463 DEBUG [RS:0;bd6592c37bb2:40087-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/f0252adfc27b54a76a05585f3f45bf90/.tmp/C/c4712c657f28446ea38507429ade8164 as hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/f0252adfc27b54a76a05585f3f45bf90/C/c4712c657f28446ea38507429ade8164 2024-11-24T22:13:55,475 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40087 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0252adfc27b54a76a05585f3f45bf90, server=bd6592c37bb2,40087,1732486423713 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-24T22:13:55,475 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40087 {}] ipc.CallRunner(138): callId: 75 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36590 deadline: 1732486495471, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0252adfc27b54a76a05585f3f45bf90, server=bd6592c37bb2,40087,1732486423713 2024-11-24T22:13:55,475 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732486426439.f0252adfc27b54a76a05585f3f45bf90., hostname=bd6592c37bb2,40087,1732486423713, seqNum=2 , the old value is region=TestAcidGuarantees,,1732486426439.f0252adfc27b54a76a05585f3f45bf90., hostname=bd6592c37bb2,40087,1732486423713, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0252adfc27b54a76a05585f3f45bf90, server=bd6592c37bb2,40087,1732486423713 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T22:13:55,475 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40087 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0252adfc27b54a76a05585f3f45bf90, server=bd6592c37bb2,40087,1732486423713 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-24T22:13:55,476 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732486426439.f0252adfc27b54a76a05585f3f45bf90., hostname=bd6592c37bb2,40087,1732486423713, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0252adfc27b54a76a05585f3f45bf90, server=bd6592c37bb2,40087,1732486423713 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T22:13:55,474 INFO [RS:0;bd6592c37bb2:40087-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in f0252adfc27b54a76a05585f3f45bf90/C of f0252adfc27b54a76a05585f3f45bf90 into c4712c657f28446ea38507429ade8164(size=12.2 K), total size for store is 12.2 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-24T22:13:55,476 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732486426439.f0252adfc27b54a76a05585f3f45bf90., hostname=bd6592c37bb2,40087,1732486423713, seqNum=2 because the exception is null or not the one we care about 2024-11-24T22:13:55,476 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40087 {}] ipc.CallRunner(138): callId: 102 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36532 deadline: 1732486495471, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0252adfc27b54a76a05585f3f45bf90, server=bd6592c37bb2,40087,1732486423713 2024-11-24T22:13:55,476 DEBUG [RS:0;bd6592c37bb2:40087-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for f0252adfc27b54a76a05585f3f45bf90: 2024-11-24T22:13:55,476 INFO [RS:0;bd6592c37bb2:40087-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732486426439.f0252adfc27b54a76a05585f3f45bf90., storeName=f0252adfc27b54a76a05585f3f45bf90/C, priority=13, startTime=1732486435320; duration=0sec 2024-11-24T22:13:55,476 DEBUG [RS:0;bd6592c37bb2:40087-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-24T22:13:55,476 DEBUG [RS:0;bd6592c37bb2:40087-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: f0252adfc27b54a76a05585f3f45bf90:C 2024-11-24T22:13:55,476 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732486426439.f0252adfc27b54a76a05585f3f45bf90., hostname=bd6592c37bb2,40087,1732486423713, seqNum=2 , the old value is region=TestAcidGuarantees,,1732486426439.f0252adfc27b54a76a05585f3f45bf90., hostname=bd6592c37bb2,40087,1732486423713, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0252adfc27b54a76a05585f3f45bf90, server=bd6592c37bb2,40087,1732486423713 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T22:13:55,476 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732486426439.f0252adfc27b54a76a05585f3f45bf90., hostname=bd6592c37bb2,40087,1732486423713, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0252adfc27b54a76a05585f3f45bf90, server=bd6592c37bb2,40087,1732486423713 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T22:13:55,477 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732486426439.f0252adfc27b54a76a05585f3f45bf90., hostname=bd6592c37bb2,40087,1732486423713, seqNum=2 because the exception is null or not the one we care about 2024-11-24T22:13:55,485 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40087 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0252adfc27b54a76a05585f3f45bf90, server=bd6592c37bb2,40087,1732486423713 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-24T22:13:55,486 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40087 {}] ipc.CallRunner(138): callId: 90 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36562 deadline: 1732486495481, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0252adfc27b54a76a05585f3f45bf90, server=bd6592c37bb2,40087,1732486423713 2024-11-24T22:13:55,486 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732486426439.f0252adfc27b54a76a05585f3f45bf90., hostname=bd6592c37bb2,40087,1732486423713, seqNum=2 , the old value is region=TestAcidGuarantees,,1732486426439.f0252adfc27b54a76a05585f3f45bf90., hostname=bd6592c37bb2,40087,1732486423713, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0252adfc27b54a76a05585f3f45bf90, server=bd6592c37bb2,40087,1732486423713 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T22:13:55,486 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732486426439.f0252adfc27b54a76a05585f3f45bf90., hostname=bd6592c37bb2,40087,1732486423713, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0252adfc27b54a76a05585f3f45bf90, server=bd6592c37bb2,40087,1732486423713 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T22:13:55,486 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732486426439.f0252adfc27b54a76a05585f3f45bf90., hostname=bd6592c37bb2,40087,1732486423713, seqNum=2 because the exception is null or not the one we care about 2024-11-24T22:13:55,487 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40087 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0252adfc27b54a76a05585f3f45bf90, server=bd6592c37bb2,40087,1732486423713 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-24T22:13:55,487 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40087 {}] ipc.CallRunner(138): callId: 133 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36518 deadline: 1732486495482, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0252adfc27b54a76a05585f3f45bf90, server=bd6592c37bb2,40087,1732486423713 2024-11-24T22:13:55,487 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40087 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0252adfc27b54a76a05585f3f45bf90, server=bd6592c37bb2,40087,1732486423713 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-24T22:13:55,488 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732486426439.f0252adfc27b54a76a05585f3f45bf90., hostname=bd6592c37bb2,40087,1732486423713, seqNum=2 , the old value is region=TestAcidGuarantees,,1732486426439.f0252adfc27b54a76a05585f3f45bf90., hostname=bd6592c37bb2,40087,1732486423713, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0252adfc27b54a76a05585f3f45bf90, server=bd6592c37bb2,40087,1732486423713 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T22:13:55,488 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732486426439.f0252adfc27b54a76a05585f3f45bf90., hostname=bd6592c37bb2,40087,1732486423713, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0252adfc27b54a76a05585f3f45bf90, server=bd6592c37bb2,40087,1732486423713 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T22:13:55,488 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40087 {}] ipc.CallRunner(138): callId: 70 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36548 deadline: 1732486495482, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0252adfc27b54a76a05585f3f45bf90, server=bd6592c37bb2,40087,1732486423713 2024-11-24T22:13:55,488 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732486426439.f0252adfc27b54a76a05585f3f45bf90., hostname=bd6592c37bb2,40087,1732486423713, seqNum=2 because the exception is null or not the one we care about 2024-11-24T22:13:55,488 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732486426439.f0252adfc27b54a76a05585f3f45bf90., hostname=bd6592c37bb2,40087,1732486423713, seqNum=2 , the old value is region=TestAcidGuarantees,,1732486426439.f0252adfc27b54a76a05585f3f45bf90., hostname=bd6592c37bb2,40087,1732486423713, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0252adfc27b54a76a05585f3f45bf90, server=bd6592c37bb2,40087,1732486423713 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T22:13:55,488 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732486426439.f0252adfc27b54a76a05585f3f45bf90., hostname=bd6592c37bb2,40087,1732486423713, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0252adfc27b54a76a05585f3f45bf90, server=bd6592c37bb2,40087,1732486423713 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T22:13:55,488 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732486426439.f0252adfc27b54a76a05585f3f45bf90., hostname=bd6592c37bb2,40087,1732486423713, seqNum=2 because the exception is null or not the one we care about 2024-11-24T22:13:55,497 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=40087 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=12 2024-11-24T22:13:55,498 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bd6592c37bb2:0-2 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732486426439.f0252adfc27b54a76a05585f3f45bf90. 2024-11-24T22:13:55,498 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bd6592c37bb2:0-2 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732486426439.f0252adfc27b54a76a05585f3f45bf90. as already flushing 2024-11-24T22:13:55,498 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bd6592c37bb2:0-2 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732486426439.f0252adfc27b54a76a05585f3f45bf90. 2024-11-24T22:13:55,498 ERROR [RS_FLUSH_OPERATIONS-regionserver/bd6592c37bb2:0-2 {event_type=RS_FLUSH_REGIONS, pid=12}] handler.RSProcedureHandler(58): pid=12 java.io.IOException: Unable to complete flush {ENCODED => f0252adfc27b54a76a05585f3f45bf90, NAME => 'TestAcidGuarantees,,1732486426439.f0252adfc27b54a76a05585f3f45bf90.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T22:13:55,498 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bd6592c37bb2:0-2 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=12 java.io.IOException: Unable to complete flush {ENCODED => f0252adfc27b54a76a05585f3f45bf90, NAME => 'TestAcidGuarantees,,1732486426439.f0252adfc27b54a76a05585f3f45bf90.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T22:13:55,499 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37701 {}] master.HMaster(4177): Remote procedure failed, pid=12 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => f0252adfc27b54a76a05585f3f45bf90, NAME => 'TestAcidGuarantees,,1732486426439.f0252adfc27b54a76a05585f3f45bf90.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => f0252adfc27b54a76a05585f3f45bf90, NAME => 'TestAcidGuarantees,,1732486426439.f0252adfc27b54a76a05585f3f45bf90.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T22:13:55,652 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=40087 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=12 2024-11-24T22:13:55,652 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bd6592c37bb2:0-0 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732486426439.f0252adfc27b54a76a05585f3f45bf90. 2024-11-24T22:13:55,652 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bd6592c37bb2:0-0 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732486426439.f0252adfc27b54a76a05585f3f45bf90. as already flushing 2024-11-24T22:13:55,652 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bd6592c37bb2:0-0 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732486426439.f0252adfc27b54a76a05585f3f45bf90. 2024-11-24T22:13:55,652 ERROR [RS_FLUSH_OPERATIONS-regionserver/bd6592c37bb2:0-0 {event_type=RS_FLUSH_REGIONS, pid=12}] handler.RSProcedureHandler(58): pid=12 java.io.IOException: Unable to complete flush {ENCODED => f0252adfc27b54a76a05585f3f45bf90, NAME => 'TestAcidGuarantees,,1732486426439.f0252adfc27b54a76a05585f3f45bf90.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T22:13:55,653 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bd6592c37bb2:0-0 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=12 java.io.IOException: Unable to complete flush {ENCODED => f0252adfc27b54a76a05585f3f45bf90, NAME => 'TestAcidGuarantees,,1732486426439.f0252adfc27b54a76a05585f3f45bf90.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T22:13:55,654 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37701 {}] master.HMaster(4177): Remote procedure failed, pid=12 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => f0252adfc27b54a76a05585f3f45bf90, NAME => 'TestAcidGuarantees,,1732486426439.f0252adfc27b54a76a05585f3f45bf90.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => f0252adfc27b54a76a05585f3f45bf90, NAME => 'TestAcidGuarantees,,1732486426439.f0252adfc27b54a76a05585f3f45bf90.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T22:13:55,660 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37701 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=11 2024-11-24T22:13:55,688 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40087 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0252adfc27b54a76a05585f3f45bf90, server=bd6592c37bb2,40087,1732486423713 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-24T22:13:55,688 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40087 {}] ipc.CallRunner(138): callId: 76 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36590 deadline: 1732486495681, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0252adfc27b54a76a05585f3f45bf90, server=bd6592c37bb2,40087,1732486423713 2024-11-24T22:13:55,688 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40087 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0252adfc27b54a76a05585f3f45bf90, server=bd6592c37bb2,40087,1732486423713 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-24T22:13:55,689 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40087 {}] ipc.CallRunner(138): callId: 103 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36532 deadline: 1732486495681, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0252adfc27b54a76a05585f3f45bf90, server=bd6592c37bb2,40087,1732486423713 2024-11-24T22:13:55,689 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732486426439.f0252adfc27b54a76a05585f3f45bf90., hostname=bd6592c37bb2,40087,1732486423713, seqNum=2 , the old value is region=TestAcidGuarantees,,1732486426439.f0252adfc27b54a76a05585f3f45bf90., hostname=bd6592c37bb2,40087,1732486423713, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0252adfc27b54a76a05585f3f45bf90, server=bd6592c37bb2,40087,1732486423713 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T22:13:55,689 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732486426439.f0252adfc27b54a76a05585f3f45bf90., hostname=bd6592c37bb2,40087,1732486423713, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0252adfc27b54a76a05585f3f45bf90, server=bd6592c37bb2,40087,1732486423713 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T22:13:55,689 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732486426439.f0252adfc27b54a76a05585f3f45bf90., hostname=bd6592c37bb2,40087,1732486423713, seqNum=2 , the old value is region=TestAcidGuarantees,,1732486426439.f0252adfc27b54a76a05585f3f45bf90., hostname=bd6592c37bb2,40087,1732486423713, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0252adfc27b54a76a05585f3f45bf90, server=bd6592c37bb2,40087,1732486423713 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T22:13:55,689 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732486426439.f0252adfc27b54a76a05585f3f45bf90., hostname=bd6592c37bb2,40087,1732486423713, seqNum=2 because the exception is null or not the one we care about 2024-11-24T22:13:55,689 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732486426439.f0252adfc27b54a76a05585f3f45bf90., hostname=bd6592c37bb2,40087,1732486423713, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0252adfc27b54a76a05585f3f45bf90, server=bd6592c37bb2,40087,1732486423713 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T22:13:55,689 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732486426439.f0252adfc27b54a76a05585f3f45bf90., hostname=bd6592c37bb2,40087,1732486423713, seqNum=2 because the exception is null or not the one we care about 2024-11-24T22:13:55,694 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40087 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0252adfc27b54a76a05585f3f45bf90, server=bd6592c37bb2,40087,1732486423713 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-24T22:13:55,694 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40087 {}] ipc.CallRunner(138): callId: 91 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36562 deadline: 1732486495691, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0252adfc27b54a76a05585f3f45bf90, server=bd6592c37bb2,40087,1732486423713 2024-11-24T22:13:55,694 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40087 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0252adfc27b54a76a05585f3f45bf90, server=bd6592c37bb2,40087,1732486423713 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-24T22:13:55,695 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40087 {}] ipc.CallRunner(138): callId: 134 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36518 deadline: 1732486495691, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0252adfc27b54a76a05585f3f45bf90, server=bd6592c37bb2,40087,1732486423713 2024-11-24T22:13:55,695 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40087 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0252adfc27b54a76a05585f3f45bf90, server=bd6592c37bb2,40087,1732486423713 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-24T22:13:55,695 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732486426439.f0252adfc27b54a76a05585f3f45bf90., hostname=bd6592c37bb2,40087,1732486423713, seqNum=2 , the old value is region=TestAcidGuarantees,,1732486426439.f0252adfc27b54a76a05585f3f45bf90., hostname=bd6592c37bb2,40087,1732486423713, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0252adfc27b54a76a05585f3f45bf90, server=bd6592c37bb2,40087,1732486423713 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T22:13:55,695 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732486426439.f0252adfc27b54a76a05585f3f45bf90., hostname=bd6592c37bb2,40087,1732486423713, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0252adfc27b54a76a05585f3f45bf90, server=bd6592c37bb2,40087,1732486423713 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T22:13:55,695 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40087 {}] ipc.CallRunner(138): callId: 71 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36548 deadline: 1732486495691, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0252adfc27b54a76a05585f3f45bf90, server=bd6592c37bb2,40087,1732486423713 2024-11-24T22:13:55,695 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732486426439.f0252adfc27b54a76a05585f3f45bf90., hostname=bd6592c37bb2,40087,1732486423713, seqNum=2 , the old value is region=TestAcidGuarantees,,1732486426439.f0252adfc27b54a76a05585f3f45bf90., hostname=bd6592c37bb2,40087,1732486423713, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0252adfc27b54a76a05585f3f45bf90, server=bd6592c37bb2,40087,1732486423713 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T22:13:55,695 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732486426439.f0252adfc27b54a76a05585f3f45bf90., hostname=bd6592c37bb2,40087,1732486423713, seqNum=2 because the exception is null or not the one we care about 2024-11-24T22:13:55,695 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732486426439.f0252adfc27b54a76a05585f3f45bf90., hostname=bd6592c37bb2,40087,1732486423713, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0252adfc27b54a76a05585f3f45bf90, server=bd6592c37bb2,40087,1732486423713 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T22:13:55,695 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732486426439.f0252adfc27b54a76a05585f3f45bf90., hostname=bd6592c37bb2,40087,1732486423713, seqNum=2 because the exception is null or not the one we care about 2024-11-24T22:13:55,695 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732486426439.f0252adfc27b54a76a05585f3f45bf90., hostname=bd6592c37bb2,40087,1732486423713, seqNum=2 , the old value is region=TestAcidGuarantees,,1732486426439.f0252adfc27b54a76a05585f3f45bf90., hostname=bd6592c37bb2,40087,1732486423713, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0252adfc27b54a76a05585f3f45bf90, server=bd6592c37bb2,40087,1732486423713 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T22:13:55,695 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732486426439.f0252adfc27b54a76a05585f3f45bf90., hostname=bd6592c37bb2,40087,1732486423713, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0252adfc27b54a76a05585f3f45bf90, server=bd6592c37bb2,40087,1732486423713 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T22:13:55,695 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732486426439.f0252adfc27b54a76a05585f3f45bf90., hostname=bd6592c37bb2,40087,1732486423713, seqNum=2 because the exception is null or not the one we care about 2024-11-24T22:13:55,783 DEBUG [RS:0;bd6592c37bb2:40087-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/f0252adfc27b54a76a05585f3f45bf90/.tmp/A/f3da0b40cc124272a32346cc906d98c5 as hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/f0252adfc27b54a76a05585f3f45bf90/A/f3da0b40cc124272a32346cc906d98c5 2024-11-24T22:13:55,791 INFO [RS:0;bd6592c37bb2:40087-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in f0252adfc27b54a76a05585f3f45bf90/A of f0252adfc27b54a76a05585f3f45bf90 into f3da0b40cc124272a32346cc906d98c5(size=12.2 K), total size for store is 12.2 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-24T22:13:55,791 DEBUG [RS:0;bd6592c37bb2:40087-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for f0252adfc27b54a76a05585f3f45bf90: 2024-11-24T22:13:55,792 INFO [RS:0;bd6592c37bb2:40087-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732486426439.f0252adfc27b54a76a05585f3f45bf90., storeName=f0252adfc27b54a76a05585f3f45bf90/A, priority=13, startTime=1732486435320; duration=0sec 2024-11-24T22:13:55,792 DEBUG [RS:0;bd6592c37bb2:40087-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-24T22:13:55,792 DEBUG [RS:0;bd6592c37bb2:40087-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: f0252adfc27b54a76a05585f3f45bf90:A 2024-11-24T22:13:55,806 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=40087 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=12 2024-11-24T22:13:55,806 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bd6592c37bb2:0-1 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732486426439.f0252adfc27b54a76a05585f3f45bf90. 2024-11-24T22:13:55,806 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bd6592c37bb2:0-1 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732486426439.f0252adfc27b54a76a05585f3f45bf90. as already flushing 2024-11-24T22:13:55,807 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bd6592c37bb2:0-1 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732486426439.f0252adfc27b54a76a05585f3f45bf90. 2024-11-24T22:13:55,807 ERROR [RS_FLUSH_OPERATIONS-regionserver/bd6592c37bb2:0-1 {event_type=RS_FLUSH_REGIONS, pid=12}] handler.RSProcedureHandler(58): pid=12 java.io.IOException: Unable to complete flush {ENCODED => f0252adfc27b54a76a05585f3f45bf90, NAME => 'TestAcidGuarantees,,1732486426439.f0252adfc27b54a76a05585f3f45bf90.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T22:13:55,807 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bd6592c37bb2:0-1 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=12 java.io.IOException: Unable to complete flush {ENCODED => f0252adfc27b54a76a05585f3f45bf90, NAME => 'TestAcidGuarantees,,1732486426439.f0252adfc27b54a76a05585f3f45bf90.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T22:13:55,807 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37701 {}] master.HMaster(4177): Remote procedure failed, pid=12 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => f0252adfc27b54a76a05585f3f45bf90, NAME => 'TestAcidGuarantees,,1732486426439.f0252adfc27b54a76a05585f3f45bf90.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => f0252adfc27b54a76a05585f3f45bf90, NAME => 'TestAcidGuarantees,,1732486426439.f0252adfc27b54a76a05585f3f45bf90.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T22:13:55,846 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=42.49 KB at sequenceid=382 (bloomFilter=true), to=hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/f0252adfc27b54a76a05585f3f45bf90/.tmp/B/7fd5a8a6e6934156aebd2bd7ed3c6562 2024-11-24T22:13:55,858 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/f0252adfc27b54a76a05585f3f45bf90/.tmp/C/f98021b20f514ebf8abbfa23a71f409c is 50, key is test_row_0/C:col10/1732486435332/Put/seqid=0 2024-11-24T22:13:55,863 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45929 is added to blk_1073741869_1045 (size=12301) 2024-11-24T22:13:55,960 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=40087 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=12 2024-11-24T22:13:55,960 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bd6592c37bb2:0-2 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732486426439.f0252adfc27b54a76a05585f3f45bf90. 2024-11-24T22:13:55,960 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bd6592c37bb2:0-2 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732486426439.f0252adfc27b54a76a05585f3f45bf90. as already flushing 2024-11-24T22:13:55,960 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bd6592c37bb2:0-2 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732486426439.f0252adfc27b54a76a05585f3f45bf90. 2024-11-24T22:13:55,960 ERROR [RS_FLUSH_OPERATIONS-regionserver/bd6592c37bb2:0-2 {event_type=RS_FLUSH_REGIONS, pid=12}] handler.RSProcedureHandler(58): pid=12 java.io.IOException: Unable to complete flush {ENCODED => f0252adfc27b54a76a05585f3f45bf90, NAME => 'TestAcidGuarantees,,1732486426439.f0252adfc27b54a76a05585f3f45bf90.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T22:13:55,961 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bd6592c37bb2:0-2 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=12 java.io.IOException: Unable to complete flush {ENCODED => f0252adfc27b54a76a05585f3f45bf90, NAME => 'TestAcidGuarantees,,1732486426439.f0252adfc27b54a76a05585f3f45bf90.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T22:13:55,961 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37701 {}] master.HMaster(4177): Remote procedure failed, pid=12 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => f0252adfc27b54a76a05585f3f45bf90, NAME => 'TestAcidGuarantees,,1732486426439.f0252adfc27b54a76a05585f3f45bf90.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => f0252adfc27b54a76a05585f3f45bf90, NAME => 'TestAcidGuarantees,,1732486426439.f0252adfc27b54a76a05585f3f45bf90.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T22:13:56,007 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40087 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0252adfc27b54a76a05585f3f45bf90, server=bd6592c37bb2,40087,1732486423713 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-24T22:13:56,007 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40087 {}] ipc.CallRunner(138): callId: 77 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36590 deadline: 1732486496001, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0252adfc27b54a76a05585f3f45bf90, server=bd6592c37bb2,40087,1732486423713 2024-11-24T22:13:56,008 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732486426439.f0252adfc27b54a76a05585f3f45bf90., hostname=bd6592c37bb2,40087,1732486423713, seqNum=2 , the old value is region=TestAcidGuarantees,,1732486426439.f0252adfc27b54a76a05585f3f45bf90., hostname=bd6592c37bb2,40087,1732486423713, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0252adfc27b54a76a05585f3f45bf90, server=bd6592c37bb2,40087,1732486423713 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T22:13:56,008 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732486426439.f0252adfc27b54a76a05585f3f45bf90., hostname=bd6592c37bb2,40087,1732486423713, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0252adfc27b54a76a05585f3f45bf90, server=bd6592c37bb2,40087,1732486423713 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T22:13:56,008 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40087 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0252adfc27b54a76a05585f3f45bf90, server=bd6592c37bb2,40087,1732486423713 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-24T22:13:56,008 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732486426439.f0252adfc27b54a76a05585f3f45bf90., hostname=bd6592c37bb2,40087,1732486423713, seqNum=2 because the exception is null or not the one we care about 2024-11-24T22:13:56,008 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40087 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0252adfc27b54a76a05585f3f45bf90, server=bd6592c37bb2,40087,1732486423713 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-24T22:13:56,008 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40087 {}] ipc.CallRunner(138): callId: 104 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36532 deadline: 1732486496001, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0252adfc27b54a76a05585f3f45bf90, server=bd6592c37bb2,40087,1732486423713 2024-11-24T22:13:56,008 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40087 {}] ipc.CallRunner(138): callId: 92 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36562 deadline: 1732486496001, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0252adfc27b54a76a05585f3f45bf90, server=bd6592c37bb2,40087,1732486423713 2024-11-24T22:13:56,008 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732486426439.f0252adfc27b54a76a05585f3f45bf90., hostname=bd6592c37bb2,40087,1732486423713, seqNum=2 , the old value is region=TestAcidGuarantees,,1732486426439.f0252adfc27b54a76a05585f3f45bf90., hostname=bd6592c37bb2,40087,1732486423713, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0252adfc27b54a76a05585f3f45bf90, server=bd6592c37bb2,40087,1732486423713 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T22:13:56,008 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732486426439.f0252adfc27b54a76a05585f3f45bf90., hostname=bd6592c37bb2,40087,1732486423713, seqNum=2 , the old value is region=TestAcidGuarantees,,1732486426439.f0252adfc27b54a76a05585f3f45bf90., hostname=bd6592c37bb2,40087,1732486423713, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0252adfc27b54a76a05585f3f45bf90, server=bd6592c37bb2,40087,1732486423713 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T22:13:56,009 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732486426439.f0252adfc27b54a76a05585f3f45bf90., hostname=bd6592c37bb2,40087,1732486423713, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0252adfc27b54a76a05585f3f45bf90, server=bd6592c37bb2,40087,1732486423713 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T22:13:56,009 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732486426439.f0252adfc27b54a76a05585f3f45bf90., hostname=bd6592c37bb2,40087,1732486423713, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0252adfc27b54a76a05585f3f45bf90, server=bd6592c37bb2,40087,1732486423713 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T22:13:56,009 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732486426439.f0252adfc27b54a76a05585f3f45bf90., hostname=bd6592c37bb2,40087,1732486423713, seqNum=2 because the exception is null or not the one we care about 2024-11-24T22:13:56,009 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732486426439.f0252adfc27b54a76a05585f3f45bf90., hostname=bd6592c37bb2,40087,1732486423713, seqNum=2 because the exception is null or not the one we care about 2024-11-24T22:13:56,009 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40087 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0252adfc27b54a76a05585f3f45bf90, server=bd6592c37bb2,40087,1732486423713 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-24T22:13:56,009 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40087 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0252adfc27b54a76a05585f3f45bf90, server=bd6592c37bb2,40087,1732486423713 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-24T22:13:56,009 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40087 {}] ipc.CallRunner(138): callId: 72 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36548 deadline: 1732486496002, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0252adfc27b54a76a05585f3f45bf90, server=bd6592c37bb2,40087,1732486423713 2024-11-24T22:13:56,009 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40087 {}] ipc.CallRunner(138): callId: 135 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36518 deadline: 1732486496001, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0252adfc27b54a76a05585f3f45bf90, server=bd6592c37bb2,40087,1732486423713 2024-11-24T22:13:56,009 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732486426439.f0252adfc27b54a76a05585f3f45bf90., hostname=bd6592c37bb2,40087,1732486423713, seqNum=2 , the old value is region=TestAcidGuarantees,,1732486426439.f0252adfc27b54a76a05585f3f45bf90., hostname=bd6592c37bb2,40087,1732486423713, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0252adfc27b54a76a05585f3f45bf90, server=bd6592c37bb2,40087,1732486423713 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T22:13:56,009 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732486426439.f0252adfc27b54a76a05585f3f45bf90., hostname=bd6592c37bb2,40087,1732486423713, seqNum=2 , the old value is region=TestAcidGuarantees,,1732486426439.f0252adfc27b54a76a05585f3f45bf90., hostname=bd6592c37bb2,40087,1732486423713, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0252adfc27b54a76a05585f3f45bf90, server=bd6592c37bb2,40087,1732486423713 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T22:13:56,009 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732486426439.f0252adfc27b54a76a05585f3f45bf90., hostname=bd6592c37bb2,40087,1732486423713, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0252adfc27b54a76a05585f3f45bf90, server=bd6592c37bb2,40087,1732486423713 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T22:13:56,009 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732486426439.f0252adfc27b54a76a05585f3f45bf90., hostname=bd6592c37bb2,40087,1732486423713, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0252adfc27b54a76a05585f3f45bf90, server=bd6592c37bb2,40087,1732486423713 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T22:13:56,009 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732486426439.f0252adfc27b54a76a05585f3f45bf90., hostname=bd6592c37bb2,40087,1732486423713, seqNum=2 because the exception is null or not the one we care about 2024-11-24T22:13:56,009 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732486426439.f0252adfc27b54a76a05585f3f45bf90., hostname=bd6592c37bb2,40087,1732486423713, seqNum=2 because the exception is null or not the one we care about 2024-11-24T22:13:56,114 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=40087 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=12 2024-11-24T22:13:56,114 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bd6592c37bb2:0-0 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732486426439.f0252adfc27b54a76a05585f3f45bf90. 2024-11-24T22:13:56,114 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bd6592c37bb2:0-0 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732486426439.f0252adfc27b54a76a05585f3f45bf90. as already flushing 2024-11-24T22:13:56,114 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bd6592c37bb2:0-0 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732486426439.f0252adfc27b54a76a05585f3f45bf90. 2024-11-24T22:13:56,114 ERROR [RS_FLUSH_OPERATIONS-regionserver/bd6592c37bb2:0-0 {event_type=RS_FLUSH_REGIONS, pid=12}] handler.RSProcedureHandler(58): pid=12 java.io.IOException: Unable to complete flush {ENCODED => f0252adfc27b54a76a05585f3f45bf90, NAME => 'TestAcidGuarantees,,1732486426439.f0252adfc27b54a76a05585f3f45bf90.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T22:13:56,115 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bd6592c37bb2:0-0 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=12 java.io.IOException: Unable to complete flush {ENCODED => f0252adfc27b54a76a05585f3f45bf90, NAME => 'TestAcidGuarantees,,1732486426439.f0252adfc27b54a76a05585f3f45bf90.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T22:13:56,116 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37701 {}] master.HMaster(4177): Remote procedure failed, pid=12 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => f0252adfc27b54a76a05585f3f45bf90, NAME => 'TestAcidGuarantees,,1732486426439.f0252adfc27b54a76a05585f3f45bf90.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => f0252adfc27b54a76a05585f3f45bf90, NAME => 'TestAcidGuarantees,,1732486426439.f0252adfc27b54a76a05585f3f45bf90.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T22:13:56,171 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37701 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=11 2024-11-24T22:13:56,265 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=42.49 KB at sequenceid=382 (bloomFilter=true), to=hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/f0252adfc27b54a76a05585f3f45bf90/.tmp/C/f98021b20f514ebf8abbfa23a71f409c 2024-11-24T22:13:56,268 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=40087 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=12 2024-11-24T22:13:56,269 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bd6592c37bb2:0-1 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732486426439.f0252adfc27b54a76a05585f3f45bf90. 2024-11-24T22:13:56,269 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bd6592c37bb2:0-1 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732486426439.f0252adfc27b54a76a05585f3f45bf90. as already flushing 2024-11-24T22:13:56,269 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bd6592c37bb2:0-1 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732486426439.f0252adfc27b54a76a05585f3f45bf90. 2024-11-24T22:13:56,269 ERROR [RS_FLUSH_OPERATIONS-regionserver/bd6592c37bb2:0-1 {event_type=RS_FLUSH_REGIONS, pid=12}] handler.RSProcedureHandler(58): pid=12 java.io.IOException: Unable to complete flush {ENCODED => f0252adfc27b54a76a05585f3f45bf90, NAME => 'TestAcidGuarantees,,1732486426439.f0252adfc27b54a76a05585f3f45bf90.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T22:13:56,269 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bd6592c37bb2:0-1 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=12 java.io.IOException: Unable to complete flush {ENCODED => f0252adfc27b54a76a05585f3f45bf90, NAME => 'TestAcidGuarantees,,1732486426439.f0252adfc27b54a76a05585f3f45bf90.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T22:13:56,270 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37701 {}] master.HMaster(4177): Remote procedure failed, pid=12 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => f0252adfc27b54a76a05585f3f45bf90, NAME => 'TestAcidGuarantees,,1732486426439.f0252adfc27b54a76a05585f3f45bf90.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => f0252adfc27b54a76a05585f3f45bf90, NAME => 'TestAcidGuarantees,,1732486426439.f0252adfc27b54a76a05585f3f45bf90.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T22:13:56,274 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/f0252adfc27b54a76a05585f3f45bf90/.tmp/A/999745bc0c3c48e495f7184a9cd014a8 as hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/f0252adfc27b54a76a05585f3f45bf90/A/999745bc0c3c48e495f7184a9cd014a8 2024-11-24T22:13:56,281 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/f0252adfc27b54a76a05585f3f45bf90/A/999745bc0c3c48e495f7184a9cd014a8, entries=200, sequenceid=382, filesize=14.4 K 2024-11-24T22:13:56,282 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/f0252adfc27b54a76a05585f3f45bf90/.tmp/B/7fd5a8a6e6934156aebd2bd7ed3c6562 as hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/f0252adfc27b54a76a05585f3f45bf90/B/7fd5a8a6e6934156aebd2bd7ed3c6562 2024-11-24T22:13:56,290 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/f0252adfc27b54a76a05585f3f45bf90/B/7fd5a8a6e6934156aebd2bd7ed3c6562, entries=150, sequenceid=382, filesize=12.0 K 2024-11-24T22:13:56,291 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/f0252adfc27b54a76a05585f3f45bf90/.tmp/C/f98021b20f514ebf8abbfa23a71f409c as hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/f0252adfc27b54a76a05585f3f45bf90/C/f98021b20f514ebf8abbfa23a71f409c 2024-11-24T22:13:56,298 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/f0252adfc27b54a76a05585f3f45bf90/C/f98021b20f514ebf8abbfa23a71f409c, entries=150, sequenceid=382, filesize=12.0 K 2024-11-24T22:13:56,299 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~127.47 KB/130530, heapSize ~357.42 KB/366000, currentSize=53.67 KB/54960 for f0252adfc27b54a76a05585f3f45bf90 in 967ms, sequenceid=382, compaction requested=false 2024-11-24T22:13:56,299 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for f0252adfc27b54a76a05585f3f45bf90: 2024-11-24T22:13:56,422 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=40087 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=12 2024-11-24T22:13:56,423 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bd6592c37bb2:0-2 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732486426439.f0252adfc27b54a76a05585f3f45bf90. 2024-11-24T22:13:56,423 INFO [RS_FLUSH_OPERATIONS-regionserver/bd6592c37bb2:0-2 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.HRegion(2902): Flushing f0252adfc27b54a76a05585f3f45bf90 3/3 column families, dataSize=53.67 KB heapSize=160.13 KB 2024-11-24T22:13:56,423 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bd6592c37bb2:0-2 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.CompactingMemStore(205): FLUSHING TO DISK f0252adfc27b54a76a05585f3f45bf90, store=A 2024-11-24T22:13:56,424 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bd6592c37bb2:0-2 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-24T22:13:56,424 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bd6592c37bb2:0-2 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.CompactingMemStore(205): FLUSHING TO DISK f0252adfc27b54a76a05585f3f45bf90, store=B 2024-11-24T22:13:56,424 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bd6592c37bb2:0-2 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-24T22:13:56,424 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bd6592c37bb2:0-2 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.CompactingMemStore(205): FLUSHING TO DISK f0252adfc27b54a76a05585f3f45bf90, store=C 2024-11-24T22:13:56,424 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bd6592c37bb2:0-2 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-24T22:13:56,430 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bd6592c37bb2:0-2 {event_type=RS_FLUSH_REGIONS, pid=12}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/f0252adfc27b54a76a05585f3f45bf90/.tmp/A/8758c2800adf4f4dba091cf4fb29d2f3 is 50, key is test_row_0/A:col10/1732486435356/Put/seqid=0 2024-11-24T22:13:56,441 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45929 is added to blk_1073741870_1046 (size=12301) 2024-11-24T22:13:56,525 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40087 {}] regionserver.HRegion(8855): Flush requested on f0252adfc27b54a76a05585f3f45bf90 2024-11-24T22:13:56,525 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732486426439.f0252adfc27b54a76a05585f3f45bf90. as already flushing 2024-11-24T22:13:56,587 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/bd6592c37bb2:0-2 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=259, cellSize=11855, totalHeapSize=27176, min timestamp=1732486436569, max timestamp=1732486436580Num uniques 859; 2024-11-24T22:13:56,587 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/bd6592c37bb2:0-9 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=259, cellSize=11855, totalHeapSize=27176, min timestamp=1732486436569, max timestamp=1732486436580Num uniques 859; 2024-11-24T22:13:56,588 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/bd6592c37bb2:0-2 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=11855, suffix heap size=117064,new segment heap size=27176 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=259 2024-11-24T22:13:56,588 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/bd6592c37bb2:0-9 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=11855, suffix heap size=117064,new segment heap size=27176 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=259 2024-11-24T22:13:56,588 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/bd6592c37bb2:0-5 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=259, cellSize=11855, totalHeapSize=27176, min timestamp=1732486436569, max timestamp=1732486436580Num uniques 859; 2024-11-24T22:13:56,588 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/bd6592c37bb2:0-5 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=11855, suffix heap size=117064,new segment heap size=27176 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=259 2024-11-24T22:13:56,620 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40087 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0252adfc27b54a76a05585f3f45bf90, server=bd6592c37bb2,40087,1732486423713 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-24T22:13:56,620 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40087 {}] ipc.CallRunner(138): callId: 79 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36548 deadline: 1732486496615, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0252adfc27b54a76a05585f3f45bf90, server=bd6592c37bb2,40087,1732486423713 2024-11-24T22:13:56,621 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732486426439.f0252adfc27b54a76a05585f3f45bf90., hostname=bd6592c37bb2,40087,1732486423713, seqNum=2 , the old value is region=TestAcidGuarantees,,1732486426439.f0252adfc27b54a76a05585f3f45bf90., hostname=bd6592c37bb2,40087,1732486423713, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0252adfc27b54a76a05585f3f45bf90, server=bd6592c37bb2,40087,1732486423713 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T22:13:56,621 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732486426439.f0252adfc27b54a76a05585f3f45bf90., hostname=bd6592c37bb2,40087,1732486423713, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0252adfc27b54a76a05585f3f45bf90, server=bd6592c37bb2,40087,1732486423713 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T22:13:56,621 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732486426439.f0252adfc27b54a76a05585f3f45bf90., hostname=bd6592c37bb2,40087,1732486423713, seqNum=2 because the exception is null or not the one we care about 2024-11-24T22:13:56,621 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40087 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0252adfc27b54a76a05585f3f45bf90, server=bd6592c37bb2,40087,1732486423713 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-24T22:13:56,621 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40087 {}] ipc.CallRunner(138): callId: 84 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36590 deadline: 1732486496617, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0252adfc27b54a76a05585f3f45bf90, server=bd6592c37bb2,40087,1732486423713 2024-11-24T22:13:56,622 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732486426439.f0252adfc27b54a76a05585f3f45bf90., hostname=bd6592c37bb2,40087,1732486423713, seqNum=2 , the old value is region=TestAcidGuarantees,,1732486426439.f0252adfc27b54a76a05585f3f45bf90., hostname=bd6592c37bb2,40087,1732486423713, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0252adfc27b54a76a05585f3f45bf90, server=bd6592c37bb2,40087,1732486423713 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T22:13:56,622 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40087 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0252adfc27b54a76a05585f3f45bf90, server=bd6592c37bb2,40087,1732486423713 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-24T22:13:56,622 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732486426439.f0252adfc27b54a76a05585f3f45bf90., hostname=bd6592c37bb2,40087,1732486423713, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0252adfc27b54a76a05585f3f45bf90, server=bd6592c37bb2,40087,1732486423713 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T22:13:56,622 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732486426439.f0252adfc27b54a76a05585f3f45bf90., hostname=bd6592c37bb2,40087,1732486423713, seqNum=2 because the exception is null or not the one we care about 2024-11-24T22:13:56,622 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40087 {}] ipc.CallRunner(138): callId: 142 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36518 deadline: 1732486496618, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0252adfc27b54a76a05585f3f45bf90, server=bd6592c37bb2,40087,1732486423713 2024-11-24T22:13:56,622 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732486426439.f0252adfc27b54a76a05585f3f45bf90., hostname=bd6592c37bb2,40087,1732486423713, seqNum=2 , the old value is region=TestAcidGuarantees,,1732486426439.f0252adfc27b54a76a05585f3f45bf90., hostname=bd6592c37bb2,40087,1732486423713, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0252adfc27b54a76a05585f3f45bf90, server=bd6592c37bb2,40087,1732486423713 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T22:13:56,622 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40087 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0252adfc27b54a76a05585f3f45bf90, server=bd6592c37bb2,40087,1732486423713 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-24T22:13:56,622 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732486426439.f0252adfc27b54a76a05585f3f45bf90., hostname=bd6592c37bb2,40087,1732486423713, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0252adfc27b54a76a05585f3f45bf90, server=bd6592c37bb2,40087,1732486423713 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T22:13:56,622 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732486426439.f0252adfc27b54a76a05585f3f45bf90., hostname=bd6592c37bb2,40087,1732486423713, seqNum=2 because the exception is null or not the one we care about 2024-11-24T22:13:56,623 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40087 {}] ipc.CallRunner(138): callId: 100 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36562 deadline: 1732486496619, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0252adfc27b54a76a05585f3f45bf90, server=bd6592c37bb2,40087,1732486423713 2024-11-24T22:13:56,623 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40087 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0252adfc27b54a76a05585f3f45bf90, server=bd6592c37bb2,40087,1732486423713 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-24T22:13:56,623 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732486426439.f0252adfc27b54a76a05585f3f45bf90., hostname=bd6592c37bb2,40087,1732486423713, seqNum=2 , the old value is region=TestAcidGuarantees,,1732486426439.f0252adfc27b54a76a05585f3f45bf90., hostname=bd6592c37bb2,40087,1732486423713, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0252adfc27b54a76a05585f3f45bf90, server=bd6592c37bb2,40087,1732486423713 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T22:13:56,623 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40087 {}] ipc.CallRunner(138): callId: 111 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36532 deadline: 1732486496620, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0252adfc27b54a76a05585f3f45bf90, server=bd6592c37bb2,40087,1732486423713 2024-11-24T22:13:56,623 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732486426439.f0252adfc27b54a76a05585f3f45bf90., hostname=bd6592c37bb2,40087,1732486423713, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0252adfc27b54a76a05585f3f45bf90, server=bd6592c37bb2,40087,1732486423713 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T22:13:56,623 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732486426439.f0252adfc27b54a76a05585f3f45bf90., hostname=bd6592c37bb2,40087,1732486423713, seqNum=2 because the exception is null or not the one we care about 2024-11-24T22:13:56,623 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732486426439.f0252adfc27b54a76a05585f3f45bf90., hostname=bd6592c37bb2,40087,1732486423713, seqNum=2 , the old value is region=TestAcidGuarantees,,1732486426439.f0252adfc27b54a76a05585f3f45bf90., hostname=bd6592c37bb2,40087,1732486423713, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0252adfc27b54a76a05585f3f45bf90, server=bd6592c37bb2,40087,1732486423713 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T22:13:56,623 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732486426439.f0252adfc27b54a76a05585f3f45bf90., hostname=bd6592c37bb2,40087,1732486423713, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0252adfc27b54a76a05585f3f45bf90, server=bd6592c37bb2,40087,1732486423713 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T22:13:56,623 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732486426439.f0252adfc27b54a76a05585f3f45bf90., hostname=bd6592c37bb2,40087,1732486423713, seqNum=2 because the exception is null or not the one we care about 2024-11-24T22:13:56,750 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40087 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0252adfc27b54a76a05585f3f45bf90, server=bd6592c37bb2,40087,1732486423713 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-24T22:13:56,750 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40087 {}] ipc.CallRunner(138): callId: 80 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36548 deadline: 1732486496731, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0252adfc27b54a76a05585f3f45bf90, server=bd6592c37bb2,40087,1732486423713 2024-11-24T22:13:56,751 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40087 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0252adfc27b54a76a05585f3f45bf90, server=bd6592c37bb2,40087,1732486423713 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-24T22:13:56,751 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732486426439.f0252adfc27b54a76a05585f3f45bf90., hostname=bd6592c37bb2,40087,1732486423713, seqNum=2 , the old value is region=TestAcidGuarantees,,1732486426439.f0252adfc27b54a76a05585f3f45bf90., hostname=bd6592c37bb2,40087,1732486423713, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0252adfc27b54a76a05585f3f45bf90, server=bd6592c37bb2,40087,1732486423713 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T22:13:56,751 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732486426439.f0252adfc27b54a76a05585f3f45bf90., hostname=bd6592c37bb2,40087,1732486423713, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0252adfc27b54a76a05585f3f45bf90, server=bd6592c37bb2,40087,1732486423713 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T22:13:56,751 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732486426439.f0252adfc27b54a76a05585f3f45bf90., hostname=bd6592c37bb2,40087,1732486423713, seqNum=2 because the exception is null or not the one we care about 2024-11-24T22:13:56,751 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40087 {}] ipc.CallRunner(138): callId: 85 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36590 deadline: 1732486496731, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0252adfc27b54a76a05585f3f45bf90, server=bd6592c37bb2,40087,1732486423713 2024-11-24T22:13:56,752 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732486426439.f0252adfc27b54a76a05585f3f45bf90., hostname=bd6592c37bb2,40087,1732486423713, seqNum=2 , the old value is region=TestAcidGuarantees,,1732486426439.f0252adfc27b54a76a05585f3f45bf90., hostname=bd6592c37bb2,40087,1732486423713, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0252adfc27b54a76a05585f3f45bf90, server=bd6592c37bb2,40087,1732486423713 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T22:13:56,752 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732486426439.f0252adfc27b54a76a05585f3f45bf90., hostname=bd6592c37bb2,40087,1732486423713, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0252adfc27b54a76a05585f3f45bf90, server=bd6592c37bb2,40087,1732486423713 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T22:13:56,752 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40087 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0252adfc27b54a76a05585f3f45bf90, server=bd6592c37bb2,40087,1732486423713 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-24T22:13:56,752 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732486426439.f0252adfc27b54a76a05585f3f45bf90., hostname=bd6592c37bb2,40087,1732486423713, seqNum=2 because the exception is null or not the one we care about 2024-11-24T22:13:56,752 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40087 {}] ipc.CallRunner(138): callId: 143 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36518 deadline: 1732486496731, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0252adfc27b54a76a05585f3f45bf90, server=bd6592c37bb2,40087,1732486423713 2024-11-24T22:13:56,752 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732486426439.f0252adfc27b54a76a05585f3f45bf90., hostname=bd6592c37bb2,40087,1732486423713, seqNum=2 , the old value is region=TestAcidGuarantees,,1732486426439.f0252adfc27b54a76a05585f3f45bf90., hostname=bd6592c37bb2,40087,1732486423713, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0252adfc27b54a76a05585f3f45bf90, server=bd6592c37bb2,40087,1732486423713 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T22:13:56,752 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40087 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0252adfc27b54a76a05585f3f45bf90, server=bd6592c37bb2,40087,1732486423713 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-24T22:13:56,752 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732486426439.f0252adfc27b54a76a05585f3f45bf90., hostname=bd6592c37bb2,40087,1732486423713, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0252adfc27b54a76a05585f3f45bf90, server=bd6592c37bb2,40087,1732486423713 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T22:13:56,752 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732486426439.f0252adfc27b54a76a05585f3f45bf90., hostname=bd6592c37bb2,40087,1732486423713, seqNum=2 because the exception is null or not the one we care about 2024-11-24T22:13:56,753 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40087 {}] ipc.CallRunner(138): callId: 101 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36562 deadline: 1732486496731, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0252adfc27b54a76a05585f3f45bf90, server=bd6592c37bb2,40087,1732486423713 2024-11-24T22:13:56,753 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40087 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0252adfc27b54a76a05585f3f45bf90, server=bd6592c37bb2,40087,1732486423713 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-24T22:13:56,753 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40087 {}] ipc.CallRunner(138): callId: 112 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36532 deadline: 1732486496750, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0252adfc27b54a76a05585f3f45bf90, server=bd6592c37bb2,40087,1732486423713 2024-11-24T22:13:56,753 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732486426439.f0252adfc27b54a76a05585f3f45bf90., hostname=bd6592c37bb2,40087,1732486423713, seqNum=2 , the old value is region=TestAcidGuarantees,,1732486426439.f0252adfc27b54a76a05585f3f45bf90., hostname=bd6592c37bb2,40087,1732486423713, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0252adfc27b54a76a05585f3f45bf90, server=bd6592c37bb2,40087,1732486423713 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T22:13:56,753 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732486426439.f0252adfc27b54a76a05585f3f45bf90., hostname=bd6592c37bb2,40087,1732486423713, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0252adfc27b54a76a05585f3f45bf90, server=bd6592c37bb2,40087,1732486423713 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T22:13:56,753 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732486426439.f0252adfc27b54a76a05585f3f45bf90., hostname=bd6592c37bb2,40087,1732486423713, seqNum=2 because the exception is null or not the one we care about 2024-11-24T22:13:56,753 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732486426439.f0252adfc27b54a76a05585f3f45bf90., hostname=bd6592c37bb2,40087,1732486423713, seqNum=2 , the old value is region=TestAcidGuarantees,,1732486426439.f0252adfc27b54a76a05585f3f45bf90., hostname=bd6592c37bb2,40087,1732486423713, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0252adfc27b54a76a05585f3f45bf90, server=bd6592c37bb2,40087,1732486423713 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T22:13:56,753 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732486426439.f0252adfc27b54a76a05585f3f45bf90., hostname=bd6592c37bb2,40087,1732486423713, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0252adfc27b54a76a05585f3f45bf90, server=bd6592c37bb2,40087,1732486423713 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T22:13:56,753 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732486426439.f0252adfc27b54a76a05585f3f45bf90., hostname=bd6592c37bb2,40087,1732486423713, seqNum=2 because the exception is null or not the one we care about 2024-11-24T22:13:56,842 INFO [RS_FLUSH_OPERATIONS-regionserver/bd6592c37bb2:0-2 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=396 (bloomFilter=true), to=hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/f0252adfc27b54a76a05585f3f45bf90/.tmp/A/8758c2800adf4f4dba091cf4fb29d2f3 2024-11-24T22:13:56,859 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bd6592c37bb2:0-2 {event_type=RS_FLUSH_REGIONS, pid=12}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/f0252adfc27b54a76a05585f3f45bf90/.tmp/B/2e8f14af0ec94e6bb8136e2bdd679ca3 is 50, key is test_row_0/B:col10/1732486435356/Put/seqid=0 2024-11-24T22:13:56,866 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45929 is added to blk_1073741871_1047 (size=12301) 2024-11-24T22:13:56,966 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40087 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0252adfc27b54a76a05585f3f45bf90, server=bd6592c37bb2,40087,1732486423713 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-24T22:13:56,966 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40087 {}] ipc.CallRunner(138): callId: 81 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36548 deadline: 1732486496961, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0252adfc27b54a76a05585f3f45bf90, server=bd6592c37bb2,40087,1732486423713 2024-11-24T22:13:56,967 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732486426439.f0252adfc27b54a76a05585f3f45bf90., hostname=bd6592c37bb2,40087,1732486423713, seqNum=2 , the old value is region=TestAcidGuarantees,,1732486426439.f0252adfc27b54a76a05585f3f45bf90., hostname=bd6592c37bb2,40087,1732486423713, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0252adfc27b54a76a05585f3f45bf90, server=bd6592c37bb2,40087,1732486423713 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T22:13:56,967 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732486426439.f0252adfc27b54a76a05585f3f45bf90., hostname=bd6592c37bb2,40087,1732486423713, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0252adfc27b54a76a05585f3f45bf90, server=bd6592c37bb2,40087,1732486423713 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T22:13:56,967 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40087 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0252adfc27b54a76a05585f3f45bf90, server=bd6592c37bb2,40087,1732486423713 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-24T22:13:56,967 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732486426439.f0252adfc27b54a76a05585f3f45bf90., hostname=bd6592c37bb2,40087,1732486423713, seqNum=2 because the exception is null or not the one we care about 2024-11-24T22:13:56,967 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40087 {}] ipc.CallRunner(138): callId: 86 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36590 deadline: 1732486496962, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0252adfc27b54a76a05585f3f45bf90, server=bd6592c37bb2,40087,1732486423713 2024-11-24T22:13:56,967 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732486426439.f0252adfc27b54a76a05585f3f45bf90., hostname=bd6592c37bb2,40087,1732486423713, seqNum=2 , the old value is region=TestAcidGuarantees,,1732486426439.f0252adfc27b54a76a05585f3f45bf90., hostname=bd6592c37bb2,40087,1732486423713, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0252adfc27b54a76a05585f3f45bf90, server=bd6592c37bb2,40087,1732486423713 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T22:13:56,967 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732486426439.f0252adfc27b54a76a05585f3f45bf90., hostname=bd6592c37bb2,40087,1732486423713, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0252adfc27b54a76a05585f3f45bf90, server=bd6592c37bb2,40087,1732486423713 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T22:13:56,967 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732486426439.f0252adfc27b54a76a05585f3f45bf90., hostname=bd6592c37bb2,40087,1732486423713, seqNum=2 because the exception is null or not the one we care about 2024-11-24T22:13:56,967 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40087 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0252adfc27b54a76a05585f3f45bf90, server=bd6592c37bb2,40087,1732486423713 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-24T22:13:56,967 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40087 {}] ipc.CallRunner(138): callId: 102 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36562 deadline: 1732486496962, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0252adfc27b54a76a05585f3f45bf90, server=bd6592c37bb2,40087,1732486423713 2024-11-24T22:13:56,968 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732486426439.f0252adfc27b54a76a05585f3f45bf90., hostname=bd6592c37bb2,40087,1732486423713, seqNum=2 , the old value is region=TestAcidGuarantees,,1732486426439.f0252adfc27b54a76a05585f3f45bf90., hostname=bd6592c37bb2,40087,1732486423713, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0252adfc27b54a76a05585f3f45bf90, server=bd6592c37bb2,40087,1732486423713 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T22:13:56,968 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732486426439.f0252adfc27b54a76a05585f3f45bf90., hostname=bd6592c37bb2,40087,1732486423713, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0252adfc27b54a76a05585f3f45bf90, server=bd6592c37bb2,40087,1732486423713 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T22:13:56,968 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732486426439.f0252adfc27b54a76a05585f3f45bf90., hostname=bd6592c37bb2,40087,1732486423713, seqNum=2 because the exception is null or not the one we care about 2024-11-24T22:13:56,968 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40087 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0252adfc27b54a76a05585f3f45bf90, server=bd6592c37bb2,40087,1732486423713 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-24T22:13:56,968 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40087 {}] ipc.CallRunner(138): callId: 113 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36532 deadline: 1732486496962, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0252adfc27b54a76a05585f3f45bf90, server=bd6592c37bb2,40087,1732486423713 2024-11-24T22:13:56,968 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732486426439.f0252adfc27b54a76a05585f3f45bf90., hostname=bd6592c37bb2,40087,1732486423713, seqNum=2 , the old value is region=TestAcidGuarantees,,1732486426439.f0252adfc27b54a76a05585f3f45bf90., hostname=bd6592c37bb2,40087,1732486423713, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0252adfc27b54a76a05585f3f45bf90, server=bd6592c37bb2,40087,1732486423713 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T22:13:56,968 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732486426439.f0252adfc27b54a76a05585f3f45bf90., hostname=bd6592c37bb2,40087,1732486423713, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0252adfc27b54a76a05585f3f45bf90, server=bd6592c37bb2,40087,1732486423713 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T22:13:56,968 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732486426439.f0252adfc27b54a76a05585f3f45bf90., hostname=bd6592c37bb2,40087,1732486423713, seqNum=2 because the exception is null or not the one we care about 2024-11-24T22:13:56,968 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40087 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0252adfc27b54a76a05585f3f45bf90, server=bd6592c37bb2,40087,1732486423713 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-24T22:13:56,969 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40087 {}] ipc.CallRunner(138): callId: 144 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36518 deadline: 1732486496962, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0252adfc27b54a76a05585f3f45bf90, server=bd6592c37bb2,40087,1732486423713 2024-11-24T22:13:56,969 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732486426439.f0252adfc27b54a76a05585f3f45bf90., hostname=bd6592c37bb2,40087,1732486423713, seqNum=2 , the old value is region=TestAcidGuarantees,,1732486426439.f0252adfc27b54a76a05585f3f45bf90., hostname=bd6592c37bb2,40087,1732486423713, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0252adfc27b54a76a05585f3f45bf90, server=bd6592c37bb2,40087,1732486423713 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T22:13:56,969 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732486426439.f0252adfc27b54a76a05585f3f45bf90., hostname=bd6592c37bb2,40087,1732486423713, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0252adfc27b54a76a05585f3f45bf90, server=bd6592c37bb2,40087,1732486423713 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T22:13:56,969 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732486426439.f0252adfc27b54a76a05585f3f45bf90., hostname=bd6592c37bb2,40087,1732486423713, seqNum=2 because the exception is null or not the one we care about 2024-11-24T22:13:57,181 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37701 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=11 2024-11-24T22:13:57,267 INFO [RS_FLUSH_OPERATIONS-regionserver/bd6592c37bb2:0-2 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=396 (bloomFilter=true), to=hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/f0252adfc27b54a76a05585f3f45bf90/.tmp/B/2e8f14af0ec94e6bb8136e2bdd679ca3 2024-11-24T22:13:57,271 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40087 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0252adfc27b54a76a05585f3f45bf90, server=bd6592c37bb2,40087,1732486423713 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-24T22:13:57,271 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40087 {}] ipc.CallRunner(138): callId: 82 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36548 deadline: 1732486497270, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0252adfc27b54a76a05585f3f45bf90, server=bd6592c37bb2,40087,1732486423713 2024-11-24T22:13:57,272 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732486426439.f0252adfc27b54a76a05585f3f45bf90., hostname=bd6592c37bb2,40087,1732486423713, seqNum=2 , the old value is region=TestAcidGuarantees,,1732486426439.f0252adfc27b54a76a05585f3f45bf90., hostname=bd6592c37bb2,40087,1732486423713, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0252adfc27b54a76a05585f3f45bf90, server=bd6592c37bb2,40087,1732486423713 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T22:13:57,272 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732486426439.f0252adfc27b54a76a05585f3f45bf90., hostname=bd6592c37bb2,40087,1732486423713, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0252adfc27b54a76a05585f3f45bf90, server=bd6592c37bb2,40087,1732486423713 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T22:13:57,272 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40087 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0252adfc27b54a76a05585f3f45bf90, server=bd6592c37bb2,40087,1732486423713 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-24T22:13:57,272 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732486426439.f0252adfc27b54a76a05585f3f45bf90., hostname=bd6592c37bb2,40087,1732486423713, seqNum=2 because the exception is null or not the one we care about 2024-11-24T22:13:57,272 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40087 {}] ipc.CallRunner(138): callId: 87 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36590 deadline: 1732486497271, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0252adfc27b54a76a05585f3f45bf90, server=bd6592c37bb2,40087,1732486423713 2024-11-24T22:13:57,272 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732486426439.f0252adfc27b54a76a05585f3f45bf90., hostname=bd6592c37bb2,40087,1732486423713, seqNum=2 , the old value is region=TestAcidGuarantees,,1732486426439.f0252adfc27b54a76a05585f3f45bf90., hostname=bd6592c37bb2,40087,1732486423713, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0252adfc27b54a76a05585f3f45bf90, server=bd6592c37bb2,40087,1732486423713 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T22:13:57,272 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732486426439.f0252adfc27b54a76a05585f3f45bf90., hostname=bd6592c37bb2,40087,1732486423713, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0252adfc27b54a76a05585f3f45bf90, server=bd6592c37bb2,40087,1732486423713 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T22:13:57,272 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732486426439.f0252adfc27b54a76a05585f3f45bf90., hostname=bd6592c37bb2,40087,1732486423713, seqNum=2 because the exception is null or not the one we care about 2024-11-24T22:13:57,282 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40087 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0252adfc27b54a76a05585f3f45bf90, server=bd6592c37bb2,40087,1732486423713 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-24T22:13:57,283 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40087 {}] ipc.CallRunner(138): callId: 103 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36562 deadline: 1732486497281, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0252adfc27b54a76a05585f3f45bf90, server=bd6592c37bb2,40087,1732486423713 2024-11-24T22:13:57,283 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732486426439.f0252adfc27b54a76a05585f3f45bf90., hostname=bd6592c37bb2,40087,1732486423713, seqNum=2 , the old value is region=TestAcidGuarantees,,1732486426439.f0252adfc27b54a76a05585f3f45bf90., hostname=bd6592c37bb2,40087,1732486423713, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0252adfc27b54a76a05585f3f45bf90, server=bd6592c37bb2,40087,1732486423713 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T22:13:57,283 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732486426439.f0252adfc27b54a76a05585f3f45bf90., hostname=bd6592c37bb2,40087,1732486423713, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0252adfc27b54a76a05585f3f45bf90, server=bd6592c37bb2,40087,1732486423713 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T22:13:57,283 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732486426439.f0252adfc27b54a76a05585f3f45bf90., hostname=bd6592c37bb2,40087,1732486423713, seqNum=2 because the exception is null or not the one we care about 2024-11-24T22:13:57,287 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40087 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0252adfc27b54a76a05585f3f45bf90, server=bd6592c37bb2,40087,1732486423713 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-24T22:13:57,287 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40087 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0252adfc27b54a76a05585f3f45bf90, server=bd6592c37bb2,40087,1732486423713 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-24T22:13:57,287 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40087 {}] ipc.CallRunner(138): callId: 145 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36518 deadline: 1732486497282, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0252adfc27b54a76a05585f3f45bf90, server=bd6592c37bb2,40087,1732486423713 2024-11-24T22:13:57,287 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40087 {}] ipc.CallRunner(138): callId: 114 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36532 deadline: 1732486497282, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0252adfc27b54a76a05585f3f45bf90, server=bd6592c37bb2,40087,1732486423713 2024-11-24T22:13:57,288 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732486426439.f0252adfc27b54a76a05585f3f45bf90., hostname=bd6592c37bb2,40087,1732486423713, seqNum=2 , the old value is region=TestAcidGuarantees,,1732486426439.f0252adfc27b54a76a05585f3f45bf90., hostname=bd6592c37bb2,40087,1732486423713, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0252adfc27b54a76a05585f3f45bf90, server=bd6592c37bb2,40087,1732486423713 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T22:13:57,288 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732486426439.f0252adfc27b54a76a05585f3f45bf90., hostname=bd6592c37bb2,40087,1732486423713, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0252adfc27b54a76a05585f3f45bf90, server=bd6592c37bb2,40087,1732486423713 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T22:13:57,288 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732486426439.f0252adfc27b54a76a05585f3f45bf90., hostname=bd6592c37bb2,40087,1732486423713, seqNum=2 because the exception is null or not the one we care about 2024-11-24T22:13:57,290 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732486426439.f0252adfc27b54a76a05585f3f45bf90., hostname=bd6592c37bb2,40087,1732486423713, seqNum=2 , the old value is region=TestAcidGuarantees,,1732486426439.f0252adfc27b54a76a05585f3f45bf90., hostname=bd6592c37bb2,40087,1732486423713, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0252adfc27b54a76a05585f3f45bf90, server=bd6592c37bb2,40087,1732486423713 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T22:13:57,290 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732486426439.f0252adfc27b54a76a05585f3f45bf90., hostname=bd6592c37bb2,40087,1732486423713, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0252adfc27b54a76a05585f3f45bf90, server=bd6592c37bb2,40087,1732486423713 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T22:13:57,290 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732486426439.f0252adfc27b54a76a05585f3f45bf90., hostname=bd6592c37bb2,40087,1732486423713, seqNum=2 because the exception is null or not the one we care about 2024-11-24T22:13:57,290 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bd6592c37bb2:0-2 {event_type=RS_FLUSH_REGIONS, pid=12}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/f0252adfc27b54a76a05585f3f45bf90/.tmp/C/55c31382a1984b16bbd575e5a6afce7b is 50, key is test_row_0/C:col10/1732486435356/Put/seqid=0 2024-11-24T22:13:57,305 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45929 is added to blk_1073741872_1048 (size=12301) 2024-11-24T22:13:57,306 INFO [RS_FLUSH_OPERATIONS-regionserver/bd6592c37bb2:0-2 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=396 (bloomFilter=true), to=hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/f0252adfc27b54a76a05585f3f45bf90/.tmp/C/55c31382a1984b16bbd575e5a6afce7b 2024-11-24T22:13:57,320 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bd6592c37bb2:0-2 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/f0252adfc27b54a76a05585f3f45bf90/.tmp/A/8758c2800adf4f4dba091cf4fb29d2f3 as hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/f0252adfc27b54a76a05585f3f45bf90/A/8758c2800adf4f4dba091cf4fb29d2f3 2024-11-24T22:13:57,328 INFO [RS_FLUSH_OPERATIONS-regionserver/bd6592c37bb2:0-2 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/f0252adfc27b54a76a05585f3f45bf90/A/8758c2800adf4f4dba091cf4fb29d2f3, entries=150, sequenceid=396, filesize=12.0 K 2024-11-24T22:13:57,330 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bd6592c37bb2:0-2 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/f0252adfc27b54a76a05585f3f45bf90/.tmp/B/2e8f14af0ec94e6bb8136e2bdd679ca3 as hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/f0252adfc27b54a76a05585f3f45bf90/B/2e8f14af0ec94e6bb8136e2bdd679ca3 2024-11-24T22:13:57,340 INFO [RS_FLUSH_OPERATIONS-regionserver/bd6592c37bb2:0-2 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/f0252adfc27b54a76a05585f3f45bf90/B/2e8f14af0ec94e6bb8136e2bdd679ca3, entries=150, sequenceid=396, filesize=12.0 K 2024-11-24T22:13:57,341 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bd6592c37bb2:0-2 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/f0252adfc27b54a76a05585f3f45bf90/.tmp/C/55c31382a1984b16bbd575e5a6afce7b as hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/f0252adfc27b54a76a05585f3f45bf90/C/55c31382a1984b16bbd575e5a6afce7b 2024-11-24T22:13:57,352 INFO [RS_FLUSH_OPERATIONS-regionserver/bd6592c37bb2:0-2 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/f0252adfc27b54a76a05585f3f45bf90/C/55c31382a1984b16bbd575e5a6afce7b, entries=150, sequenceid=396, filesize=12.0 K 2024-11-24T22:13:57,353 INFO [RS_FLUSH_OPERATIONS-regionserver/bd6592c37bb2:0-2 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.HRegion(3140): Finished flush of dataSize ~53.67 KB/54960, heapSize ~160.08 KB/163920, currentSize=127.47 KB/130530 for f0252adfc27b54a76a05585f3f45bf90 in 930ms, sequenceid=396, compaction requested=true 2024-11-24T22:13:57,354 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bd6592c37bb2:0-2 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.HRegion(2603): Flush status journal for f0252adfc27b54a76a05585f3f45bf90: 2024-11-24T22:13:57,354 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bd6592c37bb2:0-2 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732486426439.f0252adfc27b54a76a05585f3f45bf90. 2024-11-24T22:13:57,354 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bd6592c37bb2:0-2 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=12 2024-11-24T22:13:57,354 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37701 {}] master.HMaster(4169): Remote procedure done, pid=12 2024-11-24T22:13:57,359 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=12, resume processing ppid=11 2024-11-24T22:13:57,359 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=12, ppid=11, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 2.3190 sec 2024-11-24T22:13:57,362 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=11, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=11, table=TestAcidGuarantees in 2.3280 sec 2024-11-24T22:13:57,784 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40087 {}] regionserver.HRegion(8855): Flush requested on f0252adfc27b54a76a05585f3f45bf90 2024-11-24T22:13:57,784 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing f0252adfc27b54a76a05585f3f45bf90 3/3 column families, dataSize=134.18 KB heapSize=375.61 KB 2024-11-24T22:13:57,785 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK f0252adfc27b54a76a05585f3f45bf90, store=A 2024-11-24T22:13:57,785 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-24T22:13:57,785 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK f0252adfc27b54a76a05585f3f45bf90, store=B 2024-11-24T22:13:57,785 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-24T22:13:57,785 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK f0252adfc27b54a76a05585f3f45bf90, store=C 2024-11-24T22:13:57,785 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-24T22:13:57,805 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40087 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0252adfc27b54a76a05585f3f45bf90, server=bd6592c37bb2,40087,1732486423713 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-24T22:13:57,806 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40087 {}] ipc.CallRunner(138): callId: 115 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36532 deadline: 1732486497801, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0252adfc27b54a76a05585f3f45bf90, server=bd6592c37bb2,40087,1732486423713 2024-11-24T22:13:57,805 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40087 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0252adfc27b54a76a05585f3f45bf90, server=bd6592c37bb2,40087,1732486423713 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-24T22:13:57,806 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40087 {}] ipc.CallRunner(138): callId: 146 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36518 deadline: 1732486497801, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0252adfc27b54a76a05585f3f45bf90, server=bd6592c37bb2,40087,1732486423713 2024-11-24T22:13:57,806 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732486426439.f0252adfc27b54a76a05585f3f45bf90., hostname=bd6592c37bb2,40087,1732486423713, seqNum=2 , the old value is region=TestAcidGuarantees,,1732486426439.f0252adfc27b54a76a05585f3f45bf90., hostname=bd6592c37bb2,40087,1732486423713, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0252adfc27b54a76a05585f3f45bf90, server=bd6592c37bb2,40087,1732486423713 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T22:13:57,806 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732486426439.f0252adfc27b54a76a05585f3f45bf90., hostname=bd6592c37bb2,40087,1732486423713, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0252adfc27b54a76a05585f3f45bf90, server=bd6592c37bb2,40087,1732486423713 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T22:13:57,806 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732486426439.f0252adfc27b54a76a05585f3f45bf90., hostname=bd6592c37bb2,40087,1732486423713, seqNum=2 because the exception is null or not the one we care about 2024-11-24T22:13:57,806 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732486426439.f0252adfc27b54a76a05585f3f45bf90., hostname=bd6592c37bb2,40087,1732486423713, seqNum=2 , the old value is region=TestAcidGuarantees,,1732486426439.f0252adfc27b54a76a05585f3f45bf90., hostname=bd6592c37bb2,40087,1732486423713, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0252adfc27b54a76a05585f3f45bf90, server=bd6592c37bb2,40087,1732486423713 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T22:13:57,806 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732486426439.f0252adfc27b54a76a05585f3f45bf90., hostname=bd6592c37bb2,40087,1732486423713, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0252adfc27b54a76a05585f3f45bf90, server=bd6592c37bb2,40087,1732486423713 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T22:13:57,806 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40087 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0252adfc27b54a76a05585f3f45bf90, server=bd6592c37bb2,40087,1732486423713 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-24T22:13:57,806 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732486426439.f0252adfc27b54a76a05585f3f45bf90., hostname=bd6592c37bb2,40087,1732486423713, seqNum=2 because the exception is null or not the one we care about 2024-11-24T22:13:57,806 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40087 {}] ipc.CallRunner(138): callId: 91 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36590 deadline: 1732486497803, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0252adfc27b54a76a05585f3f45bf90, server=bd6592c37bb2,40087,1732486423713 2024-11-24T22:13:57,807 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732486426439.f0252adfc27b54a76a05585f3f45bf90., hostname=bd6592c37bb2,40087,1732486423713, seqNum=2 , the old value is region=TestAcidGuarantees,,1732486426439.f0252adfc27b54a76a05585f3f45bf90., hostname=bd6592c37bb2,40087,1732486423713, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0252adfc27b54a76a05585f3f45bf90, server=bd6592c37bb2,40087,1732486423713 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T22:13:57,807 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732486426439.f0252adfc27b54a76a05585f3f45bf90., hostname=bd6592c37bb2,40087,1732486423713, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0252adfc27b54a76a05585f3f45bf90, server=bd6592c37bb2,40087,1732486423713 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T22:13:57,807 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732486426439.f0252adfc27b54a76a05585f3f45bf90., hostname=bd6592c37bb2,40087,1732486423713, seqNum=2 because the exception is null or not the one we care about 2024-11-24T22:13:57,807 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/f0252adfc27b54a76a05585f3f45bf90/.tmp/A/7f4a2c729d3b4d5bb3dacfc98aacdcfc is 50, key is test_row_0/A:col10/1732486437782/Put/seqid=0 2024-11-24T22:13:57,811 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40087 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0252adfc27b54a76a05585f3f45bf90, server=bd6592c37bb2,40087,1732486423713 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-24T22:13:57,811 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40087 {}] ipc.CallRunner(138): callId: 87 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36548 deadline: 1732486497805, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0252adfc27b54a76a05585f3f45bf90, server=bd6592c37bb2,40087,1732486423713 2024-11-24T22:13:57,811 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732486426439.f0252adfc27b54a76a05585f3f45bf90., hostname=bd6592c37bb2,40087,1732486423713, seqNum=2 , the old value is region=TestAcidGuarantees,,1732486426439.f0252adfc27b54a76a05585f3f45bf90., hostname=bd6592c37bb2,40087,1732486423713, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0252adfc27b54a76a05585f3f45bf90, server=bd6592c37bb2,40087,1732486423713 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T22:13:57,811 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40087 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0252adfc27b54a76a05585f3f45bf90, server=bd6592c37bb2,40087,1732486423713 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-24T22:13:57,811 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732486426439.f0252adfc27b54a76a05585f3f45bf90., hostname=bd6592c37bb2,40087,1732486423713, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0252adfc27b54a76a05585f3f45bf90, server=bd6592c37bb2,40087,1732486423713 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T22:13:57,812 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732486426439.f0252adfc27b54a76a05585f3f45bf90., hostname=bd6592c37bb2,40087,1732486423713, seqNum=2 because the exception is null or not the one we care about 2024-11-24T22:13:57,812 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40087 {}] ipc.CallRunner(138): callId: 106 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36562 deadline: 1732486497805, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0252adfc27b54a76a05585f3f45bf90, server=bd6592c37bb2,40087,1732486423713 2024-11-24T22:13:57,812 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732486426439.f0252adfc27b54a76a05585f3f45bf90., hostname=bd6592c37bb2,40087,1732486423713, seqNum=2 , the old value is region=TestAcidGuarantees,,1732486426439.f0252adfc27b54a76a05585f3f45bf90., hostname=bd6592c37bb2,40087,1732486423713, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0252adfc27b54a76a05585f3f45bf90, server=bd6592c37bb2,40087,1732486423713 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T22:13:57,812 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732486426439.f0252adfc27b54a76a05585f3f45bf90., hostname=bd6592c37bb2,40087,1732486423713, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0252adfc27b54a76a05585f3f45bf90, server=bd6592c37bb2,40087,1732486423713 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T22:13:57,812 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732486426439.f0252adfc27b54a76a05585f3f45bf90., hostname=bd6592c37bb2,40087,1732486423713, seqNum=2 because the exception is null or not the one we care about 2024-11-24T22:13:57,817 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45929 is added to blk_1073741873_1049 (size=12301) 2024-11-24T22:13:57,913 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40087 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0252adfc27b54a76a05585f3f45bf90, server=bd6592c37bb2,40087,1732486423713 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-24T22:13:57,914 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40087 {}] ipc.CallRunner(138): callId: 92 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36590 deadline: 1732486497911, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0252adfc27b54a76a05585f3f45bf90, server=bd6592c37bb2,40087,1732486423713 2024-11-24T22:13:57,914 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732486426439.f0252adfc27b54a76a05585f3f45bf90., hostname=bd6592c37bb2,40087,1732486423713, seqNum=2 , the old value is region=TestAcidGuarantees,,1732486426439.f0252adfc27b54a76a05585f3f45bf90., hostname=bd6592c37bb2,40087,1732486423713, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0252adfc27b54a76a05585f3f45bf90, server=bd6592c37bb2,40087,1732486423713 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T22:13:57,914 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732486426439.f0252adfc27b54a76a05585f3f45bf90., hostname=bd6592c37bb2,40087,1732486423713, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0252adfc27b54a76a05585f3f45bf90, server=bd6592c37bb2,40087,1732486423713 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T22:13:57,915 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732486426439.f0252adfc27b54a76a05585f3f45bf90., hostname=bd6592c37bb2,40087,1732486423713, seqNum=2 because the exception is null or not the one we care about 2024-11-24T22:13:57,921 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40087 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0252adfc27b54a76a05585f3f45bf90, server=bd6592c37bb2,40087,1732486423713 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-24T22:13:57,922 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40087 {}] ipc.CallRunner(138): callId: 107 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36562 deadline: 1732486497921, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0252adfc27b54a76a05585f3f45bf90, server=bd6592c37bb2,40087,1732486423713 2024-11-24T22:13:57,922 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40087 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0252adfc27b54a76a05585f3f45bf90, server=bd6592c37bb2,40087,1732486423713 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-24T22:13:57,922 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40087 {}] ipc.CallRunner(138): callId: 88 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36548 deadline: 1732486497921, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0252adfc27b54a76a05585f3f45bf90, server=bd6592c37bb2,40087,1732486423713 2024-11-24T22:13:57,922 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732486426439.f0252adfc27b54a76a05585f3f45bf90., hostname=bd6592c37bb2,40087,1732486423713, seqNum=2 , the old value is region=TestAcidGuarantees,,1732486426439.f0252adfc27b54a76a05585f3f45bf90., hostname=bd6592c37bb2,40087,1732486423713, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0252adfc27b54a76a05585f3f45bf90, server=bd6592c37bb2,40087,1732486423713 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T22:13:57,923 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732486426439.f0252adfc27b54a76a05585f3f45bf90., hostname=bd6592c37bb2,40087,1732486423713, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0252adfc27b54a76a05585f3f45bf90, server=bd6592c37bb2,40087,1732486423713 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T22:13:57,923 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732486426439.f0252adfc27b54a76a05585f3f45bf90., hostname=bd6592c37bb2,40087,1732486423713, seqNum=2 because the exception is null or not the one we care about 2024-11-24T22:13:57,935 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732486426439.f0252adfc27b54a76a05585f3f45bf90., hostname=bd6592c37bb2,40087,1732486423713, seqNum=2 , the old value is region=TestAcidGuarantees,,1732486426439.f0252adfc27b54a76a05585f3f45bf90., hostname=bd6592c37bb2,40087,1732486423713, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0252adfc27b54a76a05585f3f45bf90, server=bd6592c37bb2,40087,1732486423713 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T22:13:57,935 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732486426439.f0252adfc27b54a76a05585f3f45bf90., hostname=bd6592c37bb2,40087,1732486423713, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0252adfc27b54a76a05585f3f45bf90, server=bd6592c37bb2,40087,1732486423713 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T22:13:57,935 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732486426439.f0252adfc27b54a76a05585f3f45bf90., hostname=bd6592c37bb2,40087,1732486423713, seqNum=2 because the exception is null or not the one we care about 2024-11-24T22:13:58,123 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40087 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0252adfc27b54a76a05585f3f45bf90, server=bd6592c37bb2,40087,1732486423713 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-24T22:13:58,124 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40087 {}] ipc.CallRunner(138): callId: 93 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36590 deadline: 1732486498121, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0252adfc27b54a76a05585f3f45bf90, server=bd6592c37bb2,40087,1732486423713 2024-11-24T22:13:58,124 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732486426439.f0252adfc27b54a76a05585f3f45bf90., hostname=bd6592c37bb2,40087,1732486423713, seqNum=2 , the old value is region=TestAcidGuarantees,,1732486426439.f0252adfc27b54a76a05585f3f45bf90., hostname=bd6592c37bb2,40087,1732486423713, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0252adfc27b54a76a05585f3f45bf90, server=bd6592c37bb2,40087,1732486423713 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T22:13:58,124 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732486426439.f0252adfc27b54a76a05585f3f45bf90., hostname=bd6592c37bb2,40087,1732486423713, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0252adfc27b54a76a05585f3f45bf90, server=bd6592c37bb2,40087,1732486423713 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T22:13:58,124 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732486426439.f0252adfc27b54a76a05585f3f45bf90., hostname=bd6592c37bb2,40087,1732486423713, seqNum=2 because the exception is null or not the one we care about 2024-11-24T22:13:58,137 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40087 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0252adfc27b54a76a05585f3f45bf90, server=bd6592c37bb2,40087,1732486423713 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-24T22:13:58,137 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40087 {}] ipc.CallRunner(138): callId: 108 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36562 deadline: 1732486498131, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0252adfc27b54a76a05585f3f45bf90, server=bd6592c37bb2,40087,1732486423713 2024-11-24T22:13:58,137 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732486426439.f0252adfc27b54a76a05585f3f45bf90., hostname=bd6592c37bb2,40087,1732486423713, seqNum=2 , the old value is region=TestAcidGuarantees,,1732486426439.f0252adfc27b54a76a05585f3f45bf90., hostname=bd6592c37bb2,40087,1732486423713, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0252adfc27b54a76a05585f3f45bf90, server=bd6592c37bb2,40087,1732486423713 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T22:13:58,138 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732486426439.f0252adfc27b54a76a05585f3f45bf90., hostname=bd6592c37bb2,40087,1732486423713, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0252adfc27b54a76a05585f3f45bf90, server=bd6592c37bb2,40087,1732486423713 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T22:13:58,138 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732486426439.f0252adfc27b54a76a05585f3f45bf90., hostname=bd6592c37bb2,40087,1732486423713, seqNum=2 because the exception is null or not the one we care about 2024-11-24T22:13:58,145 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40087 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0252adfc27b54a76a05585f3f45bf90, server=bd6592c37bb2,40087,1732486423713 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-24T22:13:58,145 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40087 {}] ipc.CallRunner(138): callId: 89 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36548 deadline: 1732486498141, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0252adfc27b54a76a05585f3f45bf90, server=bd6592c37bb2,40087,1732486423713 2024-11-24T22:13:58,146 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732486426439.f0252adfc27b54a76a05585f3f45bf90., hostname=bd6592c37bb2,40087,1732486423713, seqNum=2 , the old value is region=TestAcidGuarantees,,1732486426439.f0252adfc27b54a76a05585f3f45bf90., hostname=bd6592c37bb2,40087,1732486423713, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0252adfc27b54a76a05585f3f45bf90, server=bd6592c37bb2,40087,1732486423713 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T22:13:58,146 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732486426439.f0252adfc27b54a76a05585f3f45bf90., hostname=bd6592c37bb2,40087,1732486423713, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0252adfc27b54a76a05585f3f45bf90, server=bd6592c37bb2,40087,1732486423713 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T22:13:58,146 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732486426439.f0252adfc27b54a76a05585f3f45bf90., hostname=bd6592c37bb2,40087,1732486423713, seqNum=2 because the exception is null or not the one we care about 2024-11-24T22:13:58,219 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=431 (bloomFilter=true), to=hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/f0252adfc27b54a76a05585f3f45bf90/.tmp/A/7f4a2c729d3b4d5bb3dacfc98aacdcfc 2024-11-24T22:13:58,243 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/f0252adfc27b54a76a05585f3f45bf90/.tmp/B/3710459809154d628174fac723311f35 is 50, key is test_row_0/B:col10/1732486437782/Put/seqid=0 2024-11-24T22:13:58,263 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45929 is added to blk_1073741874_1050 (size=12301) 2024-11-24T22:13:58,433 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40087 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0252adfc27b54a76a05585f3f45bf90, server=bd6592c37bb2,40087,1732486423713 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-24T22:13:58,434 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40087 {}] ipc.CallRunner(138): callId: 94 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36590 deadline: 1732486498431, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0252adfc27b54a76a05585f3f45bf90, server=bd6592c37bb2,40087,1732486423713 2024-11-24T22:13:58,434 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732486426439.f0252adfc27b54a76a05585f3f45bf90., hostname=bd6592c37bb2,40087,1732486423713, seqNum=2 , the old value is region=TestAcidGuarantees,,1732486426439.f0252adfc27b54a76a05585f3f45bf90., hostname=bd6592c37bb2,40087,1732486423713, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0252adfc27b54a76a05585f3f45bf90, server=bd6592c37bb2,40087,1732486423713 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T22:13:58,434 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732486426439.f0252adfc27b54a76a05585f3f45bf90., hostname=bd6592c37bb2,40087,1732486423713, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0252adfc27b54a76a05585f3f45bf90, server=bd6592c37bb2,40087,1732486423713 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T22:13:58,435 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732486426439.f0252adfc27b54a76a05585f3f45bf90., hostname=bd6592c37bb2,40087,1732486423713, seqNum=2 because the exception is null or not the one we care about 2024-11-24T22:13:58,451 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40087 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0252adfc27b54a76a05585f3f45bf90, server=bd6592c37bb2,40087,1732486423713 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-24T22:13:58,452 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40087 {}] ipc.CallRunner(138): callId: 109 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36562 deadline: 1732486498451, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0252adfc27b54a76a05585f3f45bf90, server=bd6592c37bb2,40087,1732486423713 2024-11-24T22:13:58,452 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732486426439.f0252adfc27b54a76a05585f3f45bf90., hostname=bd6592c37bb2,40087,1732486423713, seqNum=2 , the old value is region=TestAcidGuarantees,,1732486426439.f0252adfc27b54a76a05585f3f45bf90., hostname=bd6592c37bb2,40087,1732486423713, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0252adfc27b54a76a05585f3f45bf90, server=bd6592c37bb2,40087,1732486423713 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T22:13:58,452 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40087 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0252adfc27b54a76a05585f3f45bf90, server=bd6592c37bb2,40087,1732486423713 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-24T22:13:58,452 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732486426439.f0252adfc27b54a76a05585f3f45bf90., hostname=bd6592c37bb2,40087,1732486423713, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0252adfc27b54a76a05585f3f45bf90, server=bd6592c37bb2,40087,1732486423713 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T22:13:58,452 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732486426439.f0252adfc27b54a76a05585f3f45bf90., hostname=bd6592c37bb2,40087,1732486423713, seqNum=2 because the exception is null or not the one we care about 2024-11-24T22:13:58,452 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40087 {}] ipc.CallRunner(138): callId: 90 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36548 deadline: 1732486498451, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0252adfc27b54a76a05585f3f45bf90, server=bd6592c37bb2,40087,1732486423713 2024-11-24T22:13:58,453 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732486426439.f0252adfc27b54a76a05585f3f45bf90., hostname=bd6592c37bb2,40087,1732486423713, seqNum=2 , the old value is region=TestAcidGuarantees,,1732486426439.f0252adfc27b54a76a05585f3f45bf90., hostname=bd6592c37bb2,40087,1732486423713, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0252adfc27b54a76a05585f3f45bf90, server=bd6592c37bb2,40087,1732486423713 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T22:13:58,453 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732486426439.f0252adfc27b54a76a05585f3f45bf90., hostname=bd6592c37bb2,40087,1732486423713, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0252adfc27b54a76a05585f3f45bf90, server=bd6592c37bb2,40087,1732486423713 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T22:13:58,453 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732486426439.f0252adfc27b54a76a05585f3f45bf90., hostname=bd6592c37bb2,40087,1732486423713, seqNum=2 because the exception is null or not the one we care about 2024-11-24T22:13:58,664 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=431 (bloomFilter=true), to=hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/f0252adfc27b54a76a05585f3f45bf90/.tmp/B/3710459809154d628174fac723311f35 2024-11-24T22:13:58,676 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/f0252adfc27b54a76a05585f3f45bf90/.tmp/C/0aa3f38d00954b41978dd2235eb55aa5 is 50, key is test_row_0/C:col10/1732486437782/Put/seqid=0 2024-11-24T22:13:58,682 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45929 is added to blk_1073741875_1051 (size=12301) 2024-11-24T22:13:58,683 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=431 (bloomFilter=true), to=hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/f0252adfc27b54a76a05585f3f45bf90/.tmp/C/0aa3f38d00954b41978dd2235eb55aa5 2024-11-24T22:13:58,690 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/f0252adfc27b54a76a05585f3f45bf90/.tmp/A/7f4a2c729d3b4d5bb3dacfc98aacdcfc as hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/f0252adfc27b54a76a05585f3f45bf90/A/7f4a2c729d3b4d5bb3dacfc98aacdcfc 2024-11-24T22:13:58,700 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/f0252adfc27b54a76a05585f3f45bf90/A/7f4a2c729d3b4d5bb3dacfc98aacdcfc, entries=150, sequenceid=431, filesize=12.0 K 2024-11-24T22:13:58,702 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/f0252adfc27b54a76a05585f3f45bf90/.tmp/B/3710459809154d628174fac723311f35 as hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/f0252adfc27b54a76a05585f3f45bf90/B/3710459809154d628174fac723311f35 2024-11-24T22:13:58,708 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/f0252adfc27b54a76a05585f3f45bf90/B/3710459809154d628174fac723311f35, entries=150, sequenceid=431, filesize=12.0 K 2024-11-24T22:13:58,710 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/f0252adfc27b54a76a05585f3f45bf90/.tmp/C/0aa3f38d00954b41978dd2235eb55aa5 as hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/f0252adfc27b54a76a05585f3f45bf90/C/0aa3f38d00954b41978dd2235eb55aa5 2024-11-24T22:13:58,725 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/f0252adfc27b54a76a05585f3f45bf90/C/0aa3f38d00954b41978dd2235eb55aa5, entries=150, sequenceid=431, filesize=12.0 K 2024-11-24T22:13:58,727 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~134.18 KB/137400, heapSize ~375.56 KB/384576, currentSize=53.67 KB/54960 for f0252adfc27b54a76a05585f3f45bf90 in 943ms, sequenceid=431, compaction requested=true 2024-11-24T22:13:58,727 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for f0252adfc27b54a76a05585f3f45bf90: 2024-11-24T22:13:58,727 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store f0252adfc27b54a76a05585f3f45bf90:A, priority=-2147483648, current under compaction store size is 1 2024-11-24T22:13:58,728 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-24T22:13:58,728 DEBUG [RS:0;bd6592c37bb2:40087-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-24T22:13:58,728 DEBUG [RS:0;bd6592c37bb2:40087-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-24T22:13:58,728 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store f0252adfc27b54a76a05585f3f45bf90:B, priority=-2147483648, current under compaction store size is 2 2024-11-24T22:13:58,728 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-24T22:13:58,728 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store f0252adfc27b54a76a05585f3f45bf90:C, priority=-2147483648, current under compaction store size is 3 2024-11-24T22:13:58,728 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-24T22:13:58,730 DEBUG [RS:0;bd6592c37bb2:40087-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 51802 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-24T22:13:58,730 DEBUG [RS:0;bd6592c37bb2:40087-shortCompactions-0 {}] regionserver.HStore(1541): f0252adfc27b54a76a05585f3f45bf90/A is initiating minor compaction (all files) 2024-11-24T22:13:58,730 INFO [RS:0;bd6592c37bb2:40087-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of f0252adfc27b54a76a05585f3f45bf90/A in TestAcidGuarantees,,1732486426439.f0252adfc27b54a76a05585f3f45bf90. 2024-11-24T22:13:58,730 INFO [RS:0;bd6592c37bb2:40087-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/f0252adfc27b54a76a05585f3f45bf90/A/f3da0b40cc124272a32346cc906d98c5, hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/f0252adfc27b54a76a05585f3f45bf90/A/999745bc0c3c48e495f7184a9cd014a8, hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/f0252adfc27b54a76a05585f3f45bf90/A/8758c2800adf4f4dba091cf4fb29d2f3, hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/f0252adfc27b54a76a05585f3f45bf90/A/7f4a2c729d3b4d5bb3dacfc98aacdcfc] into tmpdir=hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/f0252adfc27b54a76a05585f3f45bf90/.tmp, totalSize=50.6 K 2024-11-24T22:13:58,731 DEBUG [RS:0;bd6592c37bb2:40087-shortCompactions-0 {}] compactions.Compactor(225): Compacting f3da0b40cc124272a32346cc906d98c5, keycount=150, bloomtype=ROW, size=12.2 K, encoding=NONE, compression=NONE, seqNum=193, earliestPutTs=1732486433576 2024-11-24T22:13:58,731 DEBUG [RS:0;bd6592c37bb2:40087-shortCompactions-0 {}] compactions.Compactor(225): Compacting 999745bc0c3c48e495f7184a9cd014a8, keycount=200, bloomtype=ROW, size=14.4 K, encoding=NONE, compression=NONE, seqNum=382, earliestPutTs=1732486435312 2024-11-24T22:13:58,732 DEBUG [RS:0;bd6592c37bb2:40087-shortCompactions-0 {}] compactions.Compactor(225): Compacting 8758c2800adf4f4dba091cf4fb29d2f3, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=396, earliestPutTs=1732486435347 2024-11-24T22:13:58,732 DEBUG [RS:0;bd6592c37bb2:40087-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 49362 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-24T22:13:58,732 DEBUG [RS:0;bd6592c37bb2:40087-longCompactions-0 {}] regionserver.HStore(1541): f0252adfc27b54a76a05585f3f45bf90/B is initiating minor compaction (all files) 2024-11-24T22:13:58,732 INFO [RS:0;bd6592c37bb2:40087-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of f0252adfc27b54a76a05585f3f45bf90/B in TestAcidGuarantees,,1732486426439.f0252adfc27b54a76a05585f3f45bf90. 2024-11-24T22:13:58,732 INFO [RS:0;bd6592c37bb2:40087-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/f0252adfc27b54a76a05585f3f45bf90/B/ffc9abede25d4d48a0ffc2479587f52a, hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/f0252adfc27b54a76a05585f3f45bf90/B/7fd5a8a6e6934156aebd2bd7ed3c6562, hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/f0252adfc27b54a76a05585f3f45bf90/B/2e8f14af0ec94e6bb8136e2bdd679ca3, hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/f0252adfc27b54a76a05585f3f45bf90/B/3710459809154d628174fac723311f35] into tmpdir=hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/f0252adfc27b54a76a05585f3f45bf90/.tmp, totalSize=48.2 K 2024-11-24T22:13:58,732 DEBUG [RS:0;bd6592c37bb2:40087-shortCompactions-0 {}] compactions.Compactor(225): Compacting 7f4a2c729d3b4d5bb3dacfc98aacdcfc, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=431, earliestPutTs=1732486436617 2024-11-24T22:13:58,733 DEBUG [RS:0;bd6592c37bb2:40087-longCompactions-0 {}] compactions.Compactor(225): Compacting ffc9abede25d4d48a0ffc2479587f52a, keycount=150, bloomtype=ROW, size=12.2 K, encoding=NONE, compression=NONE, seqNum=193, earliestPutTs=1732486433576 2024-11-24T22:13:58,734 DEBUG [RS:0;bd6592c37bb2:40087-longCompactions-0 {}] compactions.Compactor(225): Compacting 7fd5a8a6e6934156aebd2bd7ed3c6562, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=382, earliestPutTs=1732486435328 2024-11-24T22:13:58,734 DEBUG [RS:0;bd6592c37bb2:40087-longCompactions-0 {}] compactions.Compactor(225): Compacting 2e8f14af0ec94e6bb8136e2bdd679ca3, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=396, earliestPutTs=1732486435347 2024-11-24T22:13:58,736 DEBUG [RS:0;bd6592c37bb2:40087-longCompactions-0 {}] compactions.Compactor(225): Compacting 3710459809154d628174fac723311f35, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=431, earliestPutTs=1732486436617 2024-11-24T22:13:58,764 INFO [RS:0;bd6592c37bb2:40087-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): f0252adfc27b54a76a05585f3f45bf90#A#compaction#39 average throughput is 2.18 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-24T22:13:58,765 INFO [RS:0;bd6592c37bb2:40087-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): f0252adfc27b54a76a05585f3f45bf90#B#compaction#40 average throughput is 2.18 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-24T22:13:58,765 DEBUG [RS:0;bd6592c37bb2:40087-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/f0252adfc27b54a76a05585f3f45bf90/.tmp/A/c2bd8d33a4064e3b8a2f0df67e949fa7 is 50, key is test_row_0/A:col10/1732486437782/Put/seqid=0 2024-11-24T22:13:58,766 DEBUG [RS:0;bd6592c37bb2:40087-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/f0252adfc27b54a76a05585f3f45bf90/.tmp/B/9e9ff2c611584faf92b0cdcb1129728e is 50, key is test_row_0/B:col10/1732486437782/Put/seqid=0 2024-11-24T22:13:58,787 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45929 is added to blk_1073741877_1053 (size=12745) 2024-11-24T22:13:58,790 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45929 is added to blk_1073741876_1052 (size=12745) 2024-11-24T22:13:58,799 DEBUG [RS:0;bd6592c37bb2:40087-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/f0252adfc27b54a76a05585f3f45bf90/.tmp/B/9e9ff2c611584faf92b0cdcb1129728e as hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/f0252adfc27b54a76a05585f3f45bf90/B/9e9ff2c611584faf92b0cdcb1129728e 2024-11-24T22:13:58,815 INFO [RS:0;bd6592c37bb2:40087-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in f0252adfc27b54a76a05585f3f45bf90/B of f0252adfc27b54a76a05585f3f45bf90 into 9e9ff2c611584faf92b0cdcb1129728e(size=12.4 K), total size for store is 12.4 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-24T22:13:58,815 DEBUG [RS:0;bd6592c37bb2:40087-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for f0252adfc27b54a76a05585f3f45bf90: 2024-11-24T22:13:58,815 INFO [RS:0;bd6592c37bb2:40087-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732486426439.f0252adfc27b54a76a05585f3f45bf90., storeName=f0252adfc27b54a76a05585f3f45bf90/B, priority=12, startTime=1732486438728; duration=0sec 2024-11-24T22:13:58,815 DEBUG [RS:0;bd6592c37bb2:40087-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-24T22:13:58,815 DEBUG [RS:0;bd6592c37bb2:40087-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: f0252adfc27b54a76a05585f3f45bf90:B 2024-11-24T22:13:58,815 DEBUG [RS:0;bd6592c37bb2:40087-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-24T22:13:58,818 DEBUG [RS:0;bd6592c37bb2:40087-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 49362 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-24T22:13:58,819 DEBUG [RS:0;bd6592c37bb2:40087-longCompactions-0 {}] regionserver.HStore(1541): f0252adfc27b54a76a05585f3f45bf90/C is initiating minor compaction (all files) 2024-11-24T22:13:58,819 INFO [RS:0;bd6592c37bb2:40087-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of f0252adfc27b54a76a05585f3f45bf90/C in TestAcidGuarantees,,1732486426439.f0252adfc27b54a76a05585f3f45bf90. 2024-11-24T22:13:58,819 INFO [RS:0;bd6592c37bb2:40087-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/f0252adfc27b54a76a05585f3f45bf90/C/c4712c657f28446ea38507429ade8164, hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/f0252adfc27b54a76a05585f3f45bf90/C/f98021b20f514ebf8abbfa23a71f409c, hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/f0252adfc27b54a76a05585f3f45bf90/C/55c31382a1984b16bbd575e5a6afce7b, hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/f0252adfc27b54a76a05585f3f45bf90/C/0aa3f38d00954b41978dd2235eb55aa5] into tmpdir=hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/f0252adfc27b54a76a05585f3f45bf90/.tmp, totalSize=48.2 K 2024-11-24T22:13:58,820 DEBUG [RS:0;bd6592c37bb2:40087-longCompactions-0 {}] compactions.Compactor(225): Compacting c4712c657f28446ea38507429ade8164, keycount=150, bloomtype=ROW, size=12.2 K, encoding=NONE, compression=NONE, seqNum=193, earliestPutTs=1732486433576 2024-11-24T22:13:58,820 DEBUG [RS:0;bd6592c37bb2:40087-longCompactions-0 {}] compactions.Compactor(225): Compacting f98021b20f514ebf8abbfa23a71f409c, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=382, earliestPutTs=1732486435328 2024-11-24T22:13:58,821 DEBUG [RS:0;bd6592c37bb2:40087-longCompactions-0 {}] compactions.Compactor(225): Compacting 55c31382a1984b16bbd575e5a6afce7b, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=396, earliestPutTs=1732486435347 2024-11-24T22:13:58,821 DEBUG [RS:0;bd6592c37bb2:40087-longCompactions-0 {}] compactions.Compactor(225): Compacting 0aa3f38d00954b41978dd2235eb55aa5, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=431, earliestPutTs=1732486436617 2024-11-24T22:13:58,832 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing f0252adfc27b54a76a05585f3f45bf90 3/3 column families, dataSize=60.38 KB heapSize=180.05 KB 2024-11-24T22:13:58,832 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK f0252adfc27b54a76a05585f3f45bf90, store=A 2024-11-24T22:13:58,832 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-24T22:13:58,832 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK f0252adfc27b54a76a05585f3f45bf90, store=B 2024-11-24T22:13:58,832 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-24T22:13:58,832 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK f0252adfc27b54a76a05585f3f45bf90, store=C 2024-11-24T22:13:58,832 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-24T22:13:58,833 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40087 {}] regionserver.HRegion(8855): Flush requested on f0252adfc27b54a76a05585f3f45bf90 2024-11-24T22:13:58,839 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/f0252adfc27b54a76a05585f3f45bf90/.tmp/A/858d340a75944cf79e54a6945720ed2e is 50, key is test_row_0/A:col10/1732486438825/Put/seqid=0 2024-11-24T22:13:58,857 INFO [RS:0;bd6592c37bb2:40087-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): f0252adfc27b54a76a05585f3f45bf90#C#compaction#42 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-24T22:13:58,858 DEBUG [RS:0;bd6592c37bb2:40087-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/f0252adfc27b54a76a05585f3f45bf90/.tmp/C/2eeb43633aa74ec58c0313ac4a1fc08b is 50, key is test_row_0/C:col10/1732486437782/Put/seqid=0 2024-11-24T22:13:58,866 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45929 is added to blk_1073741878_1054 (size=12301) 2024-11-24T22:13:58,892 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45929 is added to blk_1073741879_1055 (size=12745) 2024-11-24T22:13:58,911 DEBUG [RS:0;bd6592c37bb2:40087-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/f0252adfc27b54a76a05585f3f45bf90/.tmp/C/2eeb43633aa74ec58c0313ac4a1fc08b as hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/f0252adfc27b54a76a05585f3f45bf90/C/2eeb43633aa74ec58c0313ac4a1fc08b 2024-11-24T22:13:58,930 INFO [RS:0;bd6592c37bb2:40087-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in f0252adfc27b54a76a05585f3f45bf90/C of f0252adfc27b54a76a05585f3f45bf90 into 2eeb43633aa74ec58c0313ac4a1fc08b(size=12.4 K), total size for store is 12.4 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-24T22:13:58,930 DEBUG [RS:0;bd6592c37bb2:40087-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for f0252adfc27b54a76a05585f3f45bf90: 2024-11-24T22:13:58,930 INFO [RS:0;bd6592c37bb2:40087-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732486426439.f0252adfc27b54a76a05585f3f45bf90., storeName=f0252adfc27b54a76a05585f3f45bf90/C, priority=12, startTime=1732486438728; duration=0sec 2024-11-24T22:13:58,931 DEBUG [RS:0;bd6592c37bb2:40087-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-24T22:13:58,931 DEBUG [RS:0;bd6592c37bb2:40087-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: f0252adfc27b54a76a05585f3f45bf90:C 2024-11-24T22:13:58,932 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40087 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0252adfc27b54a76a05585f3f45bf90, server=bd6592c37bb2,40087,1732486423713 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-24T22:13:58,932 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40087 {}] ipc.CallRunner(138): callId: 125 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36532 deadline: 1732486498927, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0252adfc27b54a76a05585f3f45bf90, server=bd6592c37bb2,40087,1732486423713 2024-11-24T22:13:58,932 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732486426439.f0252adfc27b54a76a05585f3f45bf90., hostname=bd6592c37bb2,40087,1732486423713, seqNum=2 , the old value is region=TestAcidGuarantees,,1732486426439.f0252adfc27b54a76a05585f3f45bf90., hostname=bd6592c37bb2,40087,1732486423713, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0252adfc27b54a76a05585f3f45bf90, server=bd6592c37bb2,40087,1732486423713 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T22:13:58,933 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732486426439.f0252adfc27b54a76a05585f3f45bf90., hostname=bd6592c37bb2,40087,1732486423713, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0252adfc27b54a76a05585f3f45bf90, server=bd6592c37bb2,40087,1732486423713 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T22:13:58,933 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732486426439.f0252adfc27b54a76a05585f3f45bf90., hostname=bd6592c37bb2,40087,1732486423713, seqNum=2 because the exception is null or not the one we care about 2024-11-24T22:13:58,940 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40087 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0252adfc27b54a76a05585f3f45bf90, server=bd6592c37bb2,40087,1732486423713 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-24T22:13:58,940 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40087 {}] ipc.CallRunner(138): callId: 156 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36518 deadline: 1732486498933, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0252adfc27b54a76a05585f3f45bf90, server=bd6592c37bb2,40087,1732486423713 2024-11-24T22:13:58,940 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732486426439.f0252adfc27b54a76a05585f3f45bf90., hostname=bd6592c37bb2,40087,1732486423713, seqNum=2 , the old value is region=TestAcidGuarantees,,1732486426439.f0252adfc27b54a76a05585f3f45bf90., hostname=bd6592c37bb2,40087,1732486423713, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0252adfc27b54a76a05585f3f45bf90, server=bd6592c37bb2,40087,1732486423713 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T22:13:58,941 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732486426439.f0252adfc27b54a76a05585f3f45bf90., hostname=bd6592c37bb2,40087,1732486423713, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0252adfc27b54a76a05585f3f45bf90, server=bd6592c37bb2,40087,1732486423713 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T22:13:58,941 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732486426439.f0252adfc27b54a76a05585f3f45bf90., hostname=bd6592c37bb2,40087,1732486423713, seqNum=2 because the exception is null or not the one we care about 2024-11-24T22:13:58,946 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40087 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0252adfc27b54a76a05585f3f45bf90, server=bd6592c37bb2,40087,1732486423713 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-24T22:13:58,947 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40087 {}] ipc.CallRunner(138): callId: 95 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36590 deadline: 1732486498941, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0252adfc27b54a76a05585f3f45bf90, server=bd6592c37bb2,40087,1732486423713 2024-11-24T22:13:58,951 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732486426439.f0252adfc27b54a76a05585f3f45bf90., hostname=bd6592c37bb2,40087,1732486423713, seqNum=2 , the old value is region=TestAcidGuarantees,,1732486426439.f0252adfc27b54a76a05585f3f45bf90., hostname=bd6592c37bb2,40087,1732486423713, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0252adfc27b54a76a05585f3f45bf90, server=bd6592c37bb2,40087,1732486423713 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T22:13:58,951 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732486426439.f0252adfc27b54a76a05585f3f45bf90., hostname=bd6592c37bb2,40087,1732486423713, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0252adfc27b54a76a05585f3f45bf90, server=bd6592c37bb2,40087,1732486423713 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T22:13:58,952 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732486426439.f0252adfc27b54a76a05585f3f45bf90., hostname=bd6592c37bb2,40087,1732486423713, seqNum=2 because the exception is null or not the one we care about 2024-11-24T22:13:58,966 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40087 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0252adfc27b54a76a05585f3f45bf90, server=bd6592c37bb2,40087,1732486423713 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-24T22:13:58,967 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40087 {}] ipc.CallRunner(138): callId: 110 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36562 deadline: 1732486498960, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0252adfc27b54a76a05585f3f45bf90, server=bd6592c37bb2,40087,1732486423713 2024-11-24T22:13:58,967 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40087 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0252adfc27b54a76a05585f3f45bf90, server=bd6592c37bb2,40087,1732486423713 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-24T22:13:58,967 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732486426439.f0252adfc27b54a76a05585f3f45bf90., hostname=bd6592c37bb2,40087,1732486423713, seqNum=2 , the old value is region=TestAcidGuarantees,,1732486426439.f0252adfc27b54a76a05585f3f45bf90., hostname=bd6592c37bb2,40087,1732486423713, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0252adfc27b54a76a05585f3f45bf90, server=bd6592c37bb2,40087,1732486423713 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T22:13:58,967 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40087 {}] ipc.CallRunner(138): callId: 91 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36548 deadline: 1732486498961, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0252adfc27b54a76a05585f3f45bf90, server=bd6592c37bb2,40087,1732486423713 2024-11-24T22:13:58,967 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732486426439.f0252adfc27b54a76a05585f3f45bf90., hostname=bd6592c37bb2,40087,1732486423713, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0252adfc27b54a76a05585f3f45bf90, server=bd6592c37bb2,40087,1732486423713 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T22:13:58,967 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732486426439.f0252adfc27b54a76a05585f3f45bf90., hostname=bd6592c37bb2,40087,1732486423713, seqNum=2 because the exception is null or not the one we care about 2024-11-24T22:13:58,968 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732486426439.f0252adfc27b54a76a05585f3f45bf90., hostname=bd6592c37bb2,40087,1732486423713, seqNum=2 , the old value is region=TestAcidGuarantees,,1732486426439.f0252adfc27b54a76a05585f3f45bf90., hostname=bd6592c37bb2,40087,1732486423713, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0252adfc27b54a76a05585f3f45bf90, server=bd6592c37bb2,40087,1732486423713 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T22:13:58,968 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732486426439.f0252adfc27b54a76a05585f3f45bf90., hostname=bd6592c37bb2,40087,1732486423713, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0252adfc27b54a76a05585f3f45bf90, server=bd6592c37bb2,40087,1732486423713 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T22:13:58,968 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732486426439.f0252adfc27b54a76a05585f3f45bf90., hostname=bd6592c37bb2,40087,1732486423713, seqNum=2 because the exception is null or not the one we care about 2024-11-24T22:13:59,044 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40087 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0252adfc27b54a76a05585f3f45bf90, server=bd6592c37bb2,40087,1732486423713 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-24T22:13:59,045 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40087 {}] ipc.CallRunner(138): callId: 126 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36532 deadline: 1732486499043, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0252adfc27b54a76a05585f3f45bf90, server=bd6592c37bb2,40087,1732486423713 2024-11-24T22:13:59,046 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732486426439.f0252adfc27b54a76a05585f3f45bf90., hostname=bd6592c37bb2,40087,1732486423713, seqNum=2 , the old value is region=TestAcidGuarantees,,1732486426439.f0252adfc27b54a76a05585f3f45bf90., hostname=bd6592c37bb2,40087,1732486423713, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0252adfc27b54a76a05585f3f45bf90, server=bd6592c37bb2,40087,1732486423713 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T22:13:59,047 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732486426439.f0252adfc27b54a76a05585f3f45bf90., hostname=bd6592c37bb2,40087,1732486423713, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0252adfc27b54a76a05585f3f45bf90, server=bd6592c37bb2,40087,1732486423713 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T22:13:59,047 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732486426439.f0252adfc27b54a76a05585f3f45bf90., hostname=bd6592c37bb2,40087,1732486423713, seqNum=2 because the exception is null or not the one we care about 2024-11-24T22:13:59,053 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40087 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0252adfc27b54a76a05585f3f45bf90, server=bd6592c37bb2,40087,1732486423713 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-24T22:13:59,053 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40087 {}] ipc.CallRunner(138): callId: 157 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36518 deadline: 1732486499051, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0252adfc27b54a76a05585f3f45bf90, server=bd6592c37bb2,40087,1732486423713 2024-11-24T22:13:59,054 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732486426439.f0252adfc27b54a76a05585f3f45bf90., hostname=bd6592c37bb2,40087,1732486423713, seqNum=2 , the old value is region=TestAcidGuarantees,,1732486426439.f0252adfc27b54a76a05585f3f45bf90., hostname=bd6592c37bb2,40087,1732486423713, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0252adfc27b54a76a05585f3f45bf90, server=bd6592c37bb2,40087,1732486423713 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T22:13:59,054 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732486426439.f0252adfc27b54a76a05585f3f45bf90., hostname=bd6592c37bb2,40087,1732486423713, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0252adfc27b54a76a05585f3f45bf90, server=bd6592c37bb2,40087,1732486423713 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T22:13:59,054 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732486426439.f0252adfc27b54a76a05585f3f45bf90., hostname=bd6592c37bb2,40087,1732486423713, seqNum=2 because the exception is null or not the one we care about 2024-11-24T22:13:59,190 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37701 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=11 2024-11-24T22:13:59,191 INFO [RPCClient-NioEventLoopGroup-4-8 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-24T22:13:59,191 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37701 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-24T22:13:59,193 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37701 {}] procedure2.ProcedureExecutor(1139): Stored pid=13, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=13, table=TestAcidGuarantees 2024-11-24T22:13:59,194 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37701 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=13 2024-11-24T22:13:59,194 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=13, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=13, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-24T22:13:59,195 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=13, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=13, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-24T22:13:59,195 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=14, ppid=13, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-24T22:13:59,199 DEBUG [RS:0;bd6592c37bb2:40087-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/f0252adfc27b54a76a05585f3f45bf90/.tmp/A/c2bd8d33a4064e3b8a2f0df67e949fa7 as hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/f0252adfc27b54a76a05585f3f45bf90/A/c2bd8d33a4064e3b8a2f0df67e949fa7 2024-11-24T22:13:59,206 INFO [RS:0;bd6592c37bb2:40087-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in f0252adfc27b54a76a05585f3f45bf90/A of f0252adfc27b54a76a05585f3f45bf90 into c2bd8d33a4064e3b8a2f0df67e949fa7(size=12.4 K), total size for store is 12.4 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-24T22:13:59,206 DEBUG [RS:0;bd6592c37bb2:40087-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for f0252adfc27b54a76a05585f3f45bf90: 2024-11-24T22:13:59,206 INFO [RS:0;bd6592c37bb2:40087-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732486426439.f0252adfc27b54a76a05585f3f45bf90., storeName=f0252adfc27b54a76a05585f3f45bf90/A, priority=12, startTime=1732486438727; duration=0sec 2024-11-24T22:13:59,206 DEBUG [RS:0;bd6592c37bb2:40087-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-24T22:13:59,206 DEBUG [RS:0;bd6592c37bb2:40087-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: f0252adfc27b54a76a05585f3f45bf90:A 2024-11-24T22:13:59,254 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40087 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0252adfc27b54a76a05585f3f45bf90, server=bd6592c37bb2,40087,1732486423713 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-24T22:13:59,255 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40087 {}] ipc.CallRunner(138): callId: 127 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36532 deadline: 1732486499251, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0252adfc27b54a76a05585f3f45bf90, server=bd6592c37bb2,40087,1732486423713 2024-11-24T22:13:59,255 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732486426439.f0252adfc27b54a76a05585f3f45bf90., hostname=bd6592c37bb2,40087,1732486423713, seqNum=2 , the old value is region=TestAcidGuarantees,,1732486426439.f0252adfc27b54a76a05585f3f45bf90., hostname=bd6592c37bb2,40087,1732486423713, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0252adfc27b54a76a05585f3f45bf90, server=bd6592c37bb2,40087,1732486423713 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T22:13:59,255 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732486426439.f0252adfc27b54a76a05585f3f45bf90., hostname=bd6592c37bb2,40087,1732486423713, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0252adfc27b54a76a05585f3f45bf90, server=bd6592c37bb2,40087,1732486423713 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T22:13:59,255 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732486426439.f0252adfc27b54a76a05585f3f45bf90., hostname=bd6592c37bb2,40087,1732486423713, seqNum=2 because the exception is null or not the one we care about 2024-11-24T22:13:59,265 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40087 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0252adfc27b54a76a05585f3f45bf90, server=bd6592c37bb2,40087,1732486423713 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-24T22:13:59,265 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40087 {}] ipc.CallRunner(138): callId: 158 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36518 deadline: 1732486499261, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0252adfc27b54a76a05585f3f45bf90, server=bd6592c37bb2,40087,1732486423713 2024-11-24T22:13:59,266 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732486426439.f0252adfc27b54a76a05585f3f45bf90., hostname=bd6592c37bb2,40087,1732486423713, seqNum=2 , the old value is region=TestAcidGuarantees,,1732486426439.f0252adfc27b54a76a05585f3f45bf90., hostname=bd6592c37bb2,40087,1732486423713, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0252adfc27b54a76a05585f3f45bf90, server=bd6592c37bb2,40087,1732486423713 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T22:13:59,266 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732486426439.f0252adfc27b54a76a05585f3f45bf90., hostname=bd6592c37bb2,40087,1732486423713, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0252adfc27b54a76a05585f3f45bf90, server=bd6592c37bb2,40087,1732486423713 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T22:13:59,266 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732486426439.f0252adfc27b54a76a05585f3f45bf90., hostname=bd6592c37bb2,40087,1732486423713, seqNum=2 because the exception is null or not the one we care about 2024-11-24T22:13:59,267 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=445 (bloomFilter=true), to=hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/f0252adfc27b54a76a05585f3f45bf90/.tmp/A/858d340a75944cf79e54a6945720ed2e 2024-11-24T22:13:59,278 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/f0252adfc27b54a76a05585f3f45bf90/.tmp/B/a57df684ebd4407e9ca9d68f74a40615 is 50, key is test_row_0/B:col10/1732486438825/Put/seqid=0 2024-11-24T22:13:59,282 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45929 is added to blk_1073741880_1056 (size=12301) 2024-11-24T22:13:59,301 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37701 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=13 2024-11-24T22:13:59,348 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=40087 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=14 2024-11-24T22:13:59,348 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bd6592c37bb2:0-0 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732486426439.f0252adfc27b54a76a05585f3f45bf90. 2024-11-24T22:13:59,348 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bd6592c37bb2:0-0 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732486426439.f0252adfc27b54a76a05585f3f45bf90. as already flushing 2024-11-24T22:13:59,348 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bd6592c37bb2:0-0 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732486426439.f0252adfc27b54a76a05585f3f45bf90. 2024-11-24T22:13:59,348 ERROR [RS_FLUSH_OPERATIONS-regionserver/bd6592c37bb2:0-0 {event_type=RS_FLUSH_REGIONS, pid=14}] handler.RSProcedureHandler(58): pid=14 java.io.IOException: Unable to complete flush {ENCODED => f0252adfc27b54a76a05585f3f45bf90, NAME => 'TestAcidGuarantees,,1732486426439.f0252adfc27b54a76a05585f3f45bf90.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T22:13:59,349 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bd6592c37bb2:0-0 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=14 java.io.IOException: Unable to complete flush {ENCODED => f0252adfc27b54a76a05585f3f45bf90, NAME => 'TestAcidGuarantees,,1732486426439.f0252adfc27b54a76a05585f3f45bf90.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T22:13:59,349 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37701 {}] master.HMaster(4177): Remote procedure failed, pid=14 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => f0252adfc27b54a76a05585f3f45bf90, NAME => 'TestAcidGuarantees,,1732486426439.f0252adfc27b54a76a05585f3f45bf90.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => f0252adfc27b54a76a05585f3f45bf90, NAME => 'TestAcidGuarantees,,1732486426439.f0252adfc27b54a76a05585f3f45bf90.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T22:13:59,501 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=40087 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=14 2024-11-24T22:13:59,502 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bd6592c37bb2:0-1 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732486426439.f0252adfc27b54a76a05585f3f45bf90. 2024-11-24T22:13:59,502 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bd6592c37bb2:0-1 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732486426439.f0252adfc27b54a76a05585f3f45bf90. as already flushing 2024-11-24T22:13:59,502 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bd6592c37bb2:0-1 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732486426439.f0252adfc27b54a76a05585f3f45bf90. 2024-11-24T22:13:59,502 ERROR [RS_FLUSH_OPERATIONS-regionserver/bd6592c37bb2:0-1 {event_type=RS_FLUSH_REGIONS, pid=14}] handler.RSProcedureHandler(58): pid=14 java.io.IOException: Unable to complete flush {ENCODED => f0252adfc27b54a76a05585f3f45bf90, NAME => 'TestAcidGuarantees,,1732486426439.f0252adfc27b54a76a05585f3f45bf90.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T22:13:59,502 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bd6592c37bb2:0-1 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=14 java.io.IOException: Unable to complete flush {ENCODED => f0252adfc27b54a76a05585f3f45bf90, NAME => 'TestAcidGuarantees,,1732486426439.f0252adfc27b54a76a05585f3f45bf90.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T22:13:59,503 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37701 {}] master.HMaster(4177): Remote procedure failed, pid=14 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => f0252adfc27b54a76a05585f3f45bf90, NAME => 'TestAcidGuarantees,,1732486426439.f0252adfc27b54a76a05585f3f45bf90.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => f0252adfc27b54a76a05585f3f45bf90, NAME => 'TestAcidGuarantees,,1732486426439.f0252adfc27b54a76a05585f3f45bf90.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T22:13:59,510 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37701 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=13 2024-11-24T22:13:59,564 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40087 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0252adfc27b54a76a05585f3f45bf90, server=bd6592c37bb2,40087,1732486423713 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-24T22:13:59,564 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40087 {}] ipc.CallRunner(138): callId: 128 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36532 deadline: 1732486499561, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0252adfc27b54a76a05585f3f45bf90, server=bd6592c37bb2,40087,1732486423713 2024-11-24T22:13:59,564 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732486426439.f0252adfc27b54a76a05585f3f45bf90., hostname=bd6592c37bb2,40087,1732486423713, seqNum=2 , the old value is region=TestAcidGuarantees,,1732486426439.f0252adfc27b54a76a05585f3f45bf90., hostname=bd6592c37bb2,40087,1732486423713, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0252adfc27b54a76a05585f3f45bf90, server=bd6592c37bb2,40087,1732486423713 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T22:13:59,565 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732486426439.f0252adfc27b54a76a05585f3f45bf90., hostname=bd6592c37bb2,40087,1732486423713, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0252adfc27b54a76a05585f3f45bf90, server=bd6592c37bb2,40087,1732486423713 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T22:13:59,565 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732486426439.f0252adfc27b54a76a05585f3f45bf90., hostname=bd6592c37bb2,40087,1732486423713, seqNum=2 because the exception is null or not the one we care about 2024-11-24T22:13:59,576 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40087 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0252adfc27b54a76a05585f3f45bf90, server=bd6592c37bb2,40087,1732486423713 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-24T22:13:59,577 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40087 {}] ipc.CallRunner(138): callId: 159 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36518 deadline: 1732486499570, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0252adfc27b54a76a05585f3f45bf90, server=bd6592c37bb2,40087,1732486423713 2024-11-24T22:13:59,577 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732486426439.f0252adfc27b54a76a05585f3f45bf90., hostname=bd6592c37bb2,40087,1732486423713, seqNum=2 , the old value is region=TestAcidGuarantees,,1732486426439.f0252adfc27b54a76a05585f3f45bf90., hostname=bd6592c37bb2,40087,1732486423713, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0252adfc27b54a76a05585f3f45bf90, server=bd6592c37bb2,40087,1732486423713 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T22:13:59,577 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732486426439.f0252adfc27b54a76a05585f3f45bf90., hostname=bd6592c37bb2,40087,1732486423713, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0252adfc27b54a76a05585f3f45bf90, server=bd6592c37bb2,40087,1732486423713 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T22:13:59,577 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732486426439.f0252adfc27b54a76a05585f3f45bf90., hostname=bd6592c37bb2,40087,1732486423713, seqNum=2 because the exception is null or not the one we care about 2024-11-24T22:13:59,657 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=40087 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=14 2024-11-24T22:13:59,660 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bd6592c37bb2:0-2 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732486426439.f0252adfc27b54a76a05585f3f45bf90. 2024-11-24T22:13:59,660 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bd6592c37bb2:0-2 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732486426439.f0252adfc27b54a76a05585f3f45bf90. as already flushing 2024-11-24T22:13:59,660 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bd6592c37bb2:0-2 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732486426439.f0252adfc27b54a76a05585f3f45bf90. 2024-11-24T22:13:59,660 ERROR [RS_FLUSH_OPERATIONS-regionserver/bd6592c37bb2:0-2 {event_type=RS_FLUSH_REGIONS, pid=14}] handler.RSProcedureHandler(58): pid=14 java.io.IOException: Unable to complete flush {ENCODED => f0252adfc27b54a76a05585f3f45bf90, NAME => 'TestAcidGuarantees,,1732486426439.f0252adfc27b54a76a05585f3f45bf90.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T22:13:59,661 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bd6592c37bb2:0-2 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=14 java.io.IOException: Unable to complete flush {ENCODED => f0252adfc27b54a76a05585f3f45bf90, NAME => 'TestAcidGuarantees,,1732486426439.f0252adfc27b54a76a05585f3f45bf90.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T22:13:59,667 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37701 {}] master.HMaster(4177): Remote procedure failed, pid=14 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => f0252adfc27b54a76a05585f3f45bf90, NAME => 'TestAcidGuarantees,,1732486426439.f0252adfc27b54a76a05585f3f45bf90.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => f0252adfc27b54a76a05585f3f45bf90, NAME => 'TestAcidGuarantees,,1732486426439.f0252adfc27b54a76a05585f3f45bf90.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T22:13:59,683 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=445 (bloomFilter=true), to=hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/f0252adfc27b54a76a05585f3f45bf90/.tmp/B/a57df684ebd4407e9ca9d68f74a40615 2024-11-24T22:13:59,697 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/f0252adfc27b54a76a05585f3f45bf90/.tmp/C/e8d52e0fc5344af581b6bdc556bace1e is 50, key is test_row_0/C:col10/1732486438825/Put/seqid=0 2024-11-24T22:13:59,714 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45929 is added to blk_1073741881_1057 (size=12301) 2024-11-24T22:13:59,715 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=445 (bloomFilter=true), to=hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/f0252adfc27b54a76a05585f3f45bf90/.tmp/C/e8d52e0fc5344af581b6bdc556bace1e 2024-11-24T22:13:59,726 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/f0252adfc27b54a76a05585f3f45bf90/.tmp/A/858d340a75944cf79e54a6945720ed2e as hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/f0252adfc27b54a76a05585f3f45bf90/A/858d340a75944cf79e54a6945720ed2e 2024-11-24T22:13:59,735 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/f0252adfc27b54a76a05585f3f45bf90/A/858d340a75944cf79e54a6945720ed2e, entries=150, sequenceid=445, filesize=12.0 K 2024-11-24T22:13:59,737 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/f0252adfc27b54a76a05585f3f45bf90/.tmp/B/a57df684ebd4407e9ca9d68f74a40615 as hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/f0252adfc27b54a76a05585f3f45bf90/B/a57df684ebd4407e9ca9d68f74a40615 2024-11-24T22:13:59,743 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/f0252adfc27b54a76a05585f3f45bf90/B/a57df684ebd4407e9ca9d68f74a40615, entries=150, sequenceid=445, filesize=12.0 K 2024-11-24T22:13:59,745 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/f0252adfc27b54a76a05585f3f45bf90/.tmp/C/e8d52e0fc5344af581b6bdc556bace1e as hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/f0252adfc27b54a76a05585f3f45bf90/C/e8d52e0fc5344af581b6bdc556bace1e 2024-11-24T22:13:59,754 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/f0252adfc27b54a76a05585f3f45bf90/C/e8d52e0fc5344af581b6bdc556bace1e, entries=150, sequenceid=445, filesize=12.0 K 2024-11-24T22:13:59,755 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~67.09 KB/68700, heapSize ~199.92 KB/204720, currentSize=107.34 KB/109920 for f0252adfc27b54a76a05585f3f45bf90 in 924ms, sequenceid=445, compaction requested=false 2024-11-24T22:13:59,756 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for f0252adfc27b54a76a05585f3f45bf90: 2024-11-24T22:13:59,821 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37701 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=13 2024-11-24T22:13:59,824 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=40087 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=14 2024-11-24T22:13:59,824 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bd6592c37bb2:0-0 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732486426439.f0252adfc27b54a76a05585f3f45bf90. 2024-11-24T22:13:59,825 INFO [RS_FLUSH_OPERATIONS-regionserver/bd6592c37bb2:0-0 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.HRegion(2902): Flushing f0252adfc27b54a76a05585f3f45bf90 3/3 column families, dataSize=107.34 KB heapSize=319.50 KB 2024-11-24T22:13:59,825 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bd6592c37bb2:0-0 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.CompactingMemStore(205): FLUSHING TO DISK f0252adfc27b54a76a05585f3f45bf90, store=A 2024-11-24T22:13:59,825 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bd6592c37bb2:0-0 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-24T22:13:59,825 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bd6592c37bb2:0-0 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.CompactingMemStore(205): FLUSHING TO DISK f0252adfc27b54a76a05585f3f45bf90, store=B 2024-11-24T22:13:59,825 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bd6592c37bb2:0-0 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-24T22:13:59,825 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bd6592c37bb2:0-0 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.CompactingMemStore(205): FLUSHING TO DISK f0252adfc27b54a76a05585f3f45bf90, store=C 2024-11-24T22:13:59,825 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bd6592c37bb2:0-0 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-24T22:13:59,835 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bd6592c37bb2:0-0 {event_type=RS_FLUSH_REGIONS, pid=14}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/f0252adfc27b54a76a05585f3f45bf90/.tmp/A/5bcdeaa73e0f49c29d59c30322b2c997 is 50, key is test_row_0/A:col10/1732486438923/Put/seqid=0 2024-11-24T22:13:59,841 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45929 is added to blk_1073741882_1058 (size=12301) 2024-11-24T22:13:59,969 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40087 {}] regionserver.HRegion(8855): Flush requested on f0252adfc27b54a76a05585f3f45bf90 2024-11-24T22:13:59,969 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732486426439.f0252adfc27b54a76a05585f3f45bf90. as already flushing 2024-11-24T22:14:00,004 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40087 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0252adfc27b54a76a05585f3f45bf90, server=bd6592c37bb2,40087,1732486423713 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-24T22:14:00,005 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40087 {}] ipc.CallRunner(138): callId: 101 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36590 deadline: 1732486499999, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0252adfc27b54a76a05585f3f45bf90, server=bd6592c37bb2,40087,1732486423713 2024-11-24T22:14:00,005 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732486426439.f0252adfc27b54a76a05585f3f45bf90., hostname=bd6592c37bb2,40087,1732486423713, seqNum=2 , the old value is region=TestAcidGuarantees,,1732486426439.f0252adfc27b54a76a05585f3f45bf90., hostname=bd6592c37bb2,40087,1732486423713, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0252adfc27b54a76a05585f3f45bf90, server=bd6592c37bb2,40087,1732486423713 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T22:14:00,005 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732486426439.f0252adfc27b54a76a05585f3f45bf90., hostname=bd6592c37bb2,40087,1732486423713, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0252adfc27b54a76a05585f3f45bf90, server=bd6592c37bb2,40087,1732486423713 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T22:14:00,005 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732486426439.f0252adfc27b54a76a05585f3f45bf90., hostname=bd6592c37bb2,40087,1732486423713, seqNum=2 because the exception is null or not the one we care about 2024-11-24T22:14:00,013 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40087 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0252adfc27b54a76a05585f3f45bf90, server=bd6592c37bb2,40087,1732486423713 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-24T22:14:00,013 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40087 {}] ipc.CallRunner(138): callId: 95 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36548 deadline: 1732486500004, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0252adfc27b54a76a05585f3f45bf90, server=bd6592c37bb2,40087,1732486423713 2024-11-24T22:14:00,013 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40087 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0252adfc27b54a76a05585f3f45bf90, server=bd6592c37bb2,40087,1732486423713 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-24T22:14:00,014 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40087 {}] ipc.CallRunner(138): callId: 114 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36562 deadline: 1732486500004, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0252adfc27b54a76a05585f3f45bf90, server=bd6592c37bb2,40087,1732486423713 2024-11-24T22:14:00,014 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732486426439.f0252adfc27b54a76a05585f3f45bf90., hostname=bd6592c37bb2,40087,1732486423713, seqNum=2 , the old value is region=TestAcidGuarantees,,1732486426439.f0252adfc27b54a76a05585f3f45bf90., hostname=bd6592c37bb2,40087,1732486423713, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0252adfc27b54a76a05585f3f45bf90, server=bd6592c37bb2,40087,1732486423713 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T22:14:00,014 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732486426439.f0252adfc27b54a76a05585f3f45bf90., hostname=bd6592c37bb2,40087,1732486423713, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0252adfc27b54a76a05585f3f45bf90, server=bd6592c37bb2,40087,1732486423713 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T22:14:00,014 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732486426439.f0252adfc27b54a76a05585f3f45bf90., hostname=bd6592c37bb2,40087,1732486423713, seqNum=2 because the exception is null or not the one we care about 2024-11-24T22:14:00,015 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732486426439.f0252adfc27b54a76a05585f3f45bf90., hostname=bd6592c37bb2,40087,1732486423713, seqNum=2 , the old value is region=TestAcidGuarantees,,1732486426439.f0252adfc27b54a76a05585f3f45bf90., hostname=bd6592c37bb2,40087,1732486423713, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0252adfc27b54a76a05585f3f45bf90, server=bd6592c37bb2,40087,1732486423713 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T22:14:00,015 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732486426439.f0252adfc27b54a76a05585f3f45bf90., hostname=bd6592c37bb2,40087,1732486423713, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0252adfc27b54a76a05585f3f45bf90, server=bd6592c37bb2,40087,1732486423713 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T22:14:00,015 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732486426439.f0252adfc27b54a76a05585f3f45bf90., hostname=bd6592c37bb2,40087,1732486423713, seqNum=2 because the exception is null or not the one we care about 2024-11-24T22:14:00,072 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40087 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0252adfc27b54a76a05585f3f45bf90, server=bd6592c37bb2,40087,1732486423713 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-24T22:14:00,073 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40087 {}] ipc.CallRunner(138): callId: 129 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36532 deadline: 1732486500071, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0252adfc27b54a76a05585f3f45bf90, server=bd6592c37bb2,40087,1732486423713 2024-11-24T22:14:00,073 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732486426439.f0252adfc27b54a76a05585f3f45bf90., hostname=bd6592c37bb2,40087,1732486423713, seqNum=2 , the old value is region=TestAcidGuarantees,,1732486426439.f0252adfc27b54a76a05585f3f45bf90., hostname=bd6592c37bb2,40087,1732486423713, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0252adfc27b54a76a05585f3f45bf90, server=bd6592c37bb2,40087,1732486423713 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T22:14:00,073 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732486426439.f0252adfc27b54a76a05585f3f45bf90., hostname=bd6592c37bb2,40087,1732486423713, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0252adfc27b54a76a05585f3f45bf90, server=bd6592c37bb2,40087,1732486423713 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T22:14:00,073 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732486426439.f0252adfc27b54a76a05585f3f45bf90., hostname=bd6592c37bb2,40087,1732486423713, seqNum=2 because the exception is null or not the one we care about 2024-11-24T22:14:00,083 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40087 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0252adfc27b54a76a05585f3f45bf90, server=bd6592c37bb2,40087,1732486423713 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-24T22:14:00,083 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40087 {}] ipc.CallRunner(138): callId: 160 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36518 deadline: 1732486500081, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0252adfc27b54a76a05585f3f45bf90, server=bd6592c37bb2,40087,1732486423713 2024-11-24T22:14:00,083 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732486426439.f0252adfc27b54a76a05585f3f45bf90., hostname=bd6592c37bb2,40087,1732486423713, seqNum=2 , the old value is region=TestAcidGuarantees,,1732486426439.f0252adfc27b54a76a05585f3f45bf90., hostname=bd6592c37bb2,40087,1732486423713, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0252adfc27b54a76a05585f3f45bf90, server=bd6592c37bb2,40087,1732486423713 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T22:14:00,083 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732486426439.f0252adfc27b54a76a05585f3f45bf90., hostname=bd6592c37bb2,40087,1732486423713, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0252adfc27b54a76a05585f3f45bf90, server=bd6592c37bb2,40087,1732486423713 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T22:14:00,083 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732486426439.f0252adfc27b54a76a05585f3f45bf90., hostname=bd6592c37bb2,40087,1732486423713, seqNum=2 because the exception is null or not the one we care about 2024-11-24T22:14:00,114 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40087 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0252adfc27b54a76a05585f3f45bf90, server=bd6592c37bb2,40087,1732486423713 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-24T22:14:00,115 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40087 {}] ipc.CallRunner(138): callId: 102 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36590 deadline: 1732486500111, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0252adfc27b54a76a05585f3f45bf90, server=bd6592c37bb2,40087,1732486423713 2024-11-24T22:14:00,115 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732486426439.f0252adfc27b54a76a05585f3f45bf90., hostname=bd6592c37bb2,40087,1732486423713, seqNum=2 , the old value is region=TestAcidGuarantees,,1732486426439.f0252adfc27b54a76a05585f3f45bf90., hostname=bd6592c37bb2,40087,1732486423713, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0252adfc27b54a76a05585f3f45bf90, server=bd6592c37bb2,40087,1732486423713 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T22:14:00,115 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732486426439.f0252adfc27b54a76a05585f3f45bf90., hostname=bd6592c37bb2,40087,1732486423713, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0252adfc27b54a76a05585f3f45bf90, server=bd6592c37bb2,40087,1732486423713 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T22:14:00,115 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732486426439.f0252adfc27b54a76a05585f3f45bf90., hostname=bd6592c37bb2,40087,1732486423713, seqNum=2 because the exception is null or not the one we care about 2024-11-24T22:14:00,122 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40087 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0252adfc27b54a76a05585f3f45bf90, server=bd6592c37bb2,40087,1732486423713 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-24T22:14:00,123 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40087 {}] ipc.CallRunner(138): callId: 96 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36548 deadline: 1732486500121, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0252adfc27b54a76a05585f3f45bf90, server=bd6592c37bb2,40087,1732486423713 2024-11-24T22:14:00,123 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732486426439.f0252adfc27b54a76a05585f3f45bf90., hostname=bd6592c37bb2,40087,1732486423713, seqNum=2 , the old value is region=TestAcidGuarantees,,1732486426439.f0252adfc27b54a76a05585f3f45bf90., hostname=bd6592c37bb2,40087,1732486423713, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0252adfc27b54a76a05585f3f45bf90, server=bd6592c37bb2,40087,1732486423713 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T22:14:00,123 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732486426439.f0252adfc27b54a76a05585f3f45bf90., hostname=bd6592c37bb2,40087,1732486423713, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0252adfc27b54a76a05585f3f45bf90, server=bd6592c37bb2,40087,1732486423713 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T22:14:00,123 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732486426439.f0252adfc27b54a76a05585f3f45bf90., hostname=bd6592c37bb2,40087,1732486423713, seqNum=2 because the exception is null or not the one we care about 2024-11-24T22:14:00,123 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40087 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0252adfc27b54a76a05585f3f45bf90, server=bd6592c37bb2,40087,1732486423713 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-24T22:14:00,124 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40087 {}] ipc.CallRunner(138): callId: 115 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36562 deadline: 1732486500122, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0252adfc27b54a76a05585f3f45bf90, server=bd6592c37bb2,40087,1732486423713 2024-11-24T22:14:00,124 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732486426439.f0252adfc27b54a76a05585f3f45bf90., hostname=bd6592c37bb2,40087,1732486423713, seqNum=2 , the old value is region=TestAcidGuarantees,,1732486426439.f0252adfc27b54a76a05585f3f45bf90., hostname=bd6592c37bb2,40087,1732486423713, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0252adfc27b54a76a05585f3f45bf90, server=bd6592c37bb2,40087,1732486423713 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T22:14:00,124 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732486426439.f0252adfc27b54a76a05585f3f45bf90., hostname=bd6592c37bb2,40087,1732486423713, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0252adfc27b54a76a05585f3f45bf90, server=bd6592c37bb2,40087,1732486423713 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T22:14:00,124 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732486426439.f0252adfc27b54a76a05585f3f45bf90., hostname=bd6592c37bb2,40087,1732486423713, seqNum=2 because the exception is null or not the one we care about 2024-11-24T22:14:00,242 INFO [RS_FLUSH_OPERATIONS-regionserver/bd6592c37bb2:0-0 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=35.78 KB at sequenceid=466 (bloomFilter=true), to=hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/f0252adfc27b54a76a05585f3f45bf90/.tmp/A/5bcdeaa73e0f49c29d59c30322b2c997 2024-11-24T22:14:00,260 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bd6592c37bb2:0-0 {event_type=RS_FLUSH_REGIONS, pid=14}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/f0252adfc27b54a76a05585f3f45bf90/.tmp/B/56630cf0374945f290b0ada498ece426 is 50, key is test_row_0/B:col10/1732486438923/Put/seqid=0 2024-11-24T22:14:00,270 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45929 is added to blk_1073741883_1059 (size=12301) 2024-11-24T22:14:00,271 INFO [RS_FLUSH_OPERATIONS-regionserver/bd6592c37bb2:0-0 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=35.78 KB at sequenceid=466 (bloomFilter=true), to=hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/f0252adfc27b54a76a05585f3f45bf90/.tmp/B/56630cf0374945f290b0ada498ece426 2024-11-24T22:14:00,289 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bd6592c37bb2:0-0 {event_type=RS_FLUSH_REGIONS, pid=14}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/f0252adfc27b54a76a05585f3f45bf90/.tmp/C/d1accac28c8541a0b967ecefbff6faa6 is 50, key is test_row_0/C:col10/1732486438923/Put/seqid=0 2024-11-24T22:14:00,314 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45929 is added to blk_1073741884_1060 (size=12301) 2024-11-24T22:14:00,322 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40087 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0252adfc27b54a76a05585f3f45bf90, server=bd6592c37bb2,40087,1732486423713 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-24T22:14:00,322 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40087 {}] ipc.CallRunner(138): callId: 103 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36590 deadline: 1732486500321, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0252adfc27b54a76a05585f3f45bf90, server=bd6592c37bb2,40087,1732486423713 2024-11-24T22:14:00,323 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732486426439.f0252adfc27b54a76a05585f3f45bf90., hostname=bd6592c37bb2,40087,1732486423713, seqNum=2 , the old value is region=TestAcidGuarantees,,1732486426439.f0252adfc27b54a76a05585f3f45bf90., hostname=bd6592c37bb2,40087,1732486423713, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0252adfc27b54a76a05585f3f45bf90, server=bd6592c37bb2,40087,1732486423713 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T22:14:00,323 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732486426439.f0252adfc27b54a76a05585f3f45bf90., hostname=bd6592c37bb2,40087,1732486423713, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0252adfc27b54a76a05585f3f45bf90, server=bd6592c37bb2,40087,1732486423713 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T22:14:00,323 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732486426439.f0252adfc27b54a76a05585f3f45bf90., hostname=bd6592c37bb2,40087,1732486423713, seqNum=2 because the exception is null or not the one we care about 2024-11-24T22:14:00,331 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37701 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=13 2024-11-24T22:14:00,334 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40087 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0252adfc27b54a76a05585f3f45bf90, server=bd6592c37bb2,40087,1732486423713 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-24T22:14:00,334 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40087 {}] ipc.CallRunner(138): callId: 97 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36548 deadline: 1732486500331, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0252adfc27b54a76a05585f3f45bf90, server=bd6592c37bb2,40087,1732486423713 2024-11-24T22:14:00,334 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40087 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0252adfc27b54a76a05585f3f45bf90, server=bd6592c37bb2,40087,1732486423713 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-24T22:14:00,335 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40087 {}] ipc.CallRunner(138): callId: 116 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36562 deadline: 1732486500331, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0252adfc27b54a76a05585f3f45bf90, server=bd6592c37bb2,40087,1732486423713 2024-11-24T22:14:00,335 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732486426439.f0252adfc27b54a76a05585f3f45bf90., hostname=bd6592c37bb2,40087,1732486423713, seqNum=2 , the old value is region=TestAcidGuarantees,,1732486426439.f0252adfc27b54a76a05585f3f45bf90., hostname=bd6592c37bb2,40087,1732486423713, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0252adfc27b54a76a05585f3f45bf90, server=bd6592c37bb2,40087,1732486423713 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T22:14:00,335 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732486426439.f0252adfc27b54a76a05585f3f45bf90., hostname=bd6592c37bb2,40087,1732486423713, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0252adfc27b54a76a05585f3f45bf90, server=bd6592c37bb2,40087,1732486423713 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T22:14:00,335 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732486426439.f0252adfc27b54a76a05585f3f45bf90., hostname=bd6592c37bb2,40087,1732486423713, seqNum=2 because the exception is null or not the one we care about 2024-11-24T22:14:00,336 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732486426439.f0252adfc27b54a76a05585f3f45bf90., hostname=bd6592c37bb2,40087,1732486423713, seqNum=2 , the old value is region=TestAcidGuarantees,,1732486426439.f0252adfc27b54a76a05585f3f45bf90., hostname=bd6592c37bb2,40087,1732486423713, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0252adfc27b54a76a05585f3f45bf90, server=bd6592c37bb2,40087,1732486423713 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T22:14:00,336 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732486426439.f0252adfc27b54a76a05585f3f45bf90., hostname=bd6592c37bb2,40087,1732486423713, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0252adfc27b54a76a05585f3f45bf90, server=bd6592c37bb2,40087,1732486423713 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T22:14:00,336 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732486426439.f0252adfc27b54a76a05585f3f45bf90., hostname=bd6592c37bb2,40087,1732486423713, seqNum=2 because the exception is null or not the one we care about 2024-11-24T22:14:00,633 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40087 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0252adfc27b54a76a05585f3f45bf90, server=bd6592c37bb2,40087,1732486423713 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-24T22:14:00,633 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40087 {}] ipc.CallRunner(138): callId: 104 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36590 deadline: 1732486500631, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0252adfc27b54a76a05585f3f45bf90, server=bd6592c37bb2,40087,1732486423713 2024-11-24T22:14:00,634 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732486426439.f0252adfc27b54a76a05585f3f45bf90., hostname=bd6592c37bb2,40087,1732486423713, seqNum=2 , the old value is region=TestAcidGuarantees,,1732486426439.f0252adfc27b54a76a05585f3f45bf90., hostname=bd6592c37bb2,40087,1732486423713, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0252adfc27b54a76a05585f3f45bf90, server=bd6592c37bb2,40087,1732486423713 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T22:14:00,634 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732486426439.f0252adfc27b54a76a05585f3f45bf90., hostname=bd6592c37bb2,40087,1732486423713, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0252adfc27b54a76a05585f3f45bf90, server=bd6592c37bb2,40087,1732486423713 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T22:14:00,634 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732486426439.f0252adfc27b54a76a05585f3f45bf90., hostname=bd6592c37bb2,40087,1732486423713, seqNum=2 because the exception is null or not the one we care about 2024-11-24T22:14:00,642 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40087 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0252adfc27b54a76a05585f3f45bf90, server=bd6592c37bb2,40087,1732486423713 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-24T22:14:00,642 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40087 {}] ipc.CallRunner(138): callId: 117 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36562 deadline: 1732486500641, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0252adfc27b54a76a05585f3f45bf90, server=bd6592c37bb2,40087,1732486423713 2024-11-24T22:14:00,643 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732486426439.f0252adfc27b54a76a05585f3f45bf90., hostname=bd6592c37bb2,40087,1732486423713, seqNum=2 , the old value is region=TestAcidGuarantees,,1732486426439.f0252adfc27b54a76a05585f3f45bf90., hostname=bd6592c37bb2,40087,1732486423713, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0252adfc27b54a76a05585f3f45bf90, server=bd6592c37bb2,40087,1732486423713 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T22:14:00,643 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732486426439.f0252adfc27b54a76a05585f3f45bf90., hostname=bd6592c37bb2,40087,1732486423713, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0252adfc27b54a76a05585f3f45bf90, server=bd6592c37bb2,40087,1732486423713 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T22:14:00,643 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732486426439.f0252adfc27b54a76a05585f3f45bf90., hostname=bd6592c37bb2,40087,1732486423713, seqNum=2 because the exception is null or not the one we care about 2024-11-24T22:14:00,644 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40087 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0252adfc27b54a76a05585f3f45bf90, server=bd6592c37bb2,40087,1732486423713 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-24T22:14:00,644 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40087 {}] ipc.CallRunner(138): callId: 98 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36548 deadline: 1732486500641, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0252adfc27b54a76a05585f3f45bf90, server=bd6592c37bb2,40087,1732486423713 2024-11-24T22:14:00,646 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732486426439.f0252adfc27b54a76a05585f3f45bf90., hostname=bd6592c37bb2,40087,1732486423713, seqNum=2 , the old value is region=TestAcidGuarantees,,1732486426439.f0252adfc27b54a76a05585f3f45bf90., hostname=bd6592c37bb2,40087,1732486423713, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0252adfc27b54a76a05585f3f45bf90, server=bd6592c37bb2,40087,1732486423713 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T22:14:00,647 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732486426439.f0252adfc27b54a76a05585f3f45bf90., hostname=bd6592c37bb2,40087,1732486423713, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0252adfc27b54a76a05585f3f45bf90, server=bd6592c37bb2,40087,1732486423713 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T22:14:00,647 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732486426439.f0252adfc27b54a76a05585f3f45bf90., hostname=bd6592c37bb2,40087,1732486423713, seqNum=2 because the exception is null or not the one we care about 2024-11-24T22:14:00,715 INFO [RS_FLUSH_OPERATIONS-regionserver/bd6592c37bb2:0-0 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=35.78 KB at sequenceid=466 (bloomFilter=true), to=hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/f0252adfc27b54a76a05585f3f45bf90/.tmp/C/d1accac28c8541a0b967ecefbff6faa6 2024-11-24T22:14:00,724 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bd6592c37bb2:0-0 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/f0252adfc27b54a76a05585f3f45bf90/.tmp/A/5bcdeaa73e0f49c29d59c30322b2c997 as hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/f0252adfc27b54a76a05585f3f45bf90/A/5bcdeaa73e0f49c29d59c30322b2c997 2024-11-24T22:14:00,741 INFO [RS_FLUSH_OPERATIONS-regionserver/bd6592c37bb2:0-0 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/f0252adfc27b54a76a05585f3f45bf90/A/5bcdeaa73e0f49c29d59c30322b2c997, entries=150, sequenceid=466, filesize=12.0 K 2024-11-24T22:14:00,749 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bd6592c37bb2:0-0 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/f0252adfc27b54a76a05585f3f45bf90/.tmp/B/56630cf0374945f290b0ada498ece426 as hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/f0252adfc27b54a76a05585f3f45bf90/B/56630cf0374945f290b0ada498ece426 2024-11-24T22:14:00,761 INFO [RS_FLUSH_OPERATIONS-regionserver/bd6592c37bb2:0-0 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/f0252adfc27b54a76a05585f3f45bf90/B/56630cf0374945f290b0ada498ece426, entries=150, sequenceid=466, filesize=12.0 K 2024-11-24T22:14:00,762 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bd6592c37bb2:0-0 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/f0252adfc27b54a76a05585f3f45bf90/.tmp/C/d1accac28c8541a0b967ecefbff6faa6 as hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/f0252adfc27b54a76a05585f3f45bf90/C/d1accac28c8541a0b967ecefbff6faa6 2024-11-24T22:14:00,776 INFO [RS_FLUSH_OPERATIONS-regionserver/bd6592c37bb2:0-0 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/f0252adfc27b54a76a05585f3f45bf90/C/d1accac28c8541a0b967ecefbff6faa6, entries=150, sequenceid=466, filesize=12.0 K 2024-11-24T22:14:00,778 INFO [RS_FLUSH_OPERATIONS-regionserver/bd6592c37bb2:0-0 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.HRegion(3140): Finished flush of dataSize ~107.34 KB/109920, heapSize ~319.45 KB/327120, currentSize=73.80 KB/75570 for f0252adfc27b54a76a05585f3f45bf90 in 953ms, sequenceid=466, compaction requested=true 2024-11-24T22:14:00,778 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bd6592c37bb2:0-0 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.HRegion(2603): Flush status journal for f0252adfc27b54a76a05585f3f45bf90: 2024-11-24T22:14:00,778 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bd6592c37bb2:0-0 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732486426439.f0252adfc27b54a76a05585f3f45bf90. 2024-11-24T22:14:00,778 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bd6592c37bb2:0-0 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=14 2024-11-24T22:14:00,779 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37701 {}] master.HMaster(4169): Remote procedure done, pid=14 2024-11-24T22:14:00,784 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=14, resume processing ppid=13 2024-11-24T22:14:00,784 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=14, ppid=13, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.5860 sec 2024-11-24T22:14:00,787 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=13, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=13, table=TestAcidGuarantees in 1.5950 sec 2024-11-24T22:14:01,086 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40087 {}] regionserver.HRegion(8855): Flush requested on f0252adfc27b54a76a05585f3f45bf90 2024-11-24T22:14:01,086 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing f0252adfc27b54a76a05585f3f45bf90 3/3 column families, dataSize=80.51 KB heapSize=239.81 KB 2024-11-24T22:14:01,087 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK f0252adfc27b54a76a05585f3f45bf90, store=A 2024-11-24T22:14:01,087 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-24T22:14:01,087 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK f0252adfc27b54a76a05585f3f45bf90, store=B 2024-11-24T22:14:01,087 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-24T22:14:01,087 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK f0252adfc27b54a76a05585f3f45bf90, store=C 2024-11-24T22:14:01,087 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-24T22:14:01,096 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/f0252adfc27b54a76a05585f3f45bf90/.tmp/A/7160c90e6125430f851ec64126d011ae is 50, key is test_row_0/A:col10/1732486440001/Put/seqid=0 2024-11-24T22:14:01,118 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45929 is added to blk_1073741885_1061 (size=14741) 2024-11-24T22:14:01,147 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40087 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0252adfc27b54a76a05585f3f45bf90, server=bd6592c37bb2,40087,1732486423713 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-24T22:14:01,148 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40087 {}] ipc.CallRunner(138): callId: 105 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36590 deadline: 1732486501141, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0252adfc27b54a76a05585f3f45bf90, server=bd6592c37bb2,40087,1732486423713 2024-11-24T22:14:01,148 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732486426439.f0252adfc27b54a76a05585f3f45bf90., hostname=bd6592c37bb2,40087,1732486423713, seqNum=2 , the old value is region=TestAcidGuarantees,,1732486426439.f0252adfc27b54a76a05585f3f45bf90., hostname=bd6592c37bb2,40087,1732486423713, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0252adfc27b54a76a05585f3f45bf90, server=bd6592c37bb2,40087,1732486423713 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T22:14:01,148 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40087 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0252adfc27b54a76a05585f3f45bf90, server=bd6592c37bb2,40087,1732486423713 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-24T22:14:01,148 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732486426439.f0252adfc27b54a76a05585f3f45bf90., hostname=bd6592c37bb2,40087,1732486423713, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0252adfc27b54a76a05585f3f45bf90, server=bd6592c37bb2,40087,1732486423713 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T22:14:01,148 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732486426439.f0252adfc27b54a76a05585f3f45bf90., hostname=bd6592c37bb2,40087,1732486423713, seqNum=2 because the exception is null or not the one we care about 2024-11-24T22:14:01,148 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40087 {}] ipc.CallRunner(138): callId: 139 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36532 deadline: 1732486501148, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0252adfc27b54a76a05585f3f45bf90, server=bd6592c37bb2,40087,1732486423713 2024-11-24T22:14:01,149 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732486426439.f0252adfc27b54a76a05585f3f45bf90., hostname=bd6592c37bb2,40087,1732486423713, seqNum=2 , the old value is region=TestAcidGuarantees,,1732486426439.f0252adfc27b54a76a05585f3f45bf90., hostname=bd6592c37bb2,40087,1732486423713, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0252adfc27b54a76a05585f3f45bf90, server=bd6592c37bb2,40087,1732486423713 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T22:14:01,149 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40087 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0252adfc27b54a76a05585f3f45bf90, server=bd6592c37bb2,40087,1732486423713 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-24T22:14:01,149 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732486426439.f0252adfc27b54a76a05585f3f45bf90., hostname=bd6592c37bb2,40087,1732486423713, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0252adfc27b54a76a05585f3f45bf90, server=bd6592c37bb2,40087,1732486423713 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T22:14:01,149 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732486426439.f0252adfc27b54a76a05585f3f45bf90., hostname=bd6592c37bb2,40087,1732486423713, seqNum=2 because the exception is null or not the one we care about 2024-11-24T22:14:01,149 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40087 {}] ipc.CallRunner(138): callId: 168 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36518 deadline: 1732486501148, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0252adfc27b54a76a05585f3f45bf90, server=bd6592c37bb2,40087,1732486423713 2024-11-24T22:14:01,149 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732486426439.f0252adfc27b54a76a05585f3f45bf90., hostname=bd6592c37bb2,40087,1732486423713, seqNum=2 , the old value is region=TestAcidGuarantees,,1732486426439.f0252adfc27b54a76a05585f3f45bf90., hostname=bd6592c37bb2,40087,1732486423713, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0252adfc27b54a76a05585f3f45bf90, server=bd6592c37bb2,40087,1732486423713 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T22:14:01,149 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732486426439.f0252adfc27b54a76a05585f3f45bf90., hostname=bd6592c37bb2,40087,1732486423713, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0252adfc27b54a76a05585f3f45bf90, server=bd6592c37bb2,40087,1732486423713 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T22:14:01,149 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732486426439.f0252adfc27b54a76a05585f3f45bf90., hostname=bd6592c37bb2,40087,1732486423713, seqNum=2 because the exception is null or not the one we care about 2024-11-24T22:14:01,159 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40087 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0252adfc27b54a76a05585f3f45bf90, server=bd6592c37bb2,40087,1732486423713 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-24T22:14:01,159 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40087 {}] ipc.CallRunner(138): callId: 118 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36562 deadline: 1732486501151, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0252adfc27b54a76a05585f3f45bf90, server=bd6592c37bb2,40087,1732486423713 2024-11-24T22:14:01,160 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732486426439.f0252adfc27b54a76a05585f3f45bf90., hostname=bd6592c37bb2,40087,1732486423713, seqNum=2 , the old value is region=TestAcidGuarantees,,1732486426439.f0252adfc27b54a76a05585f3f45bf90., hostname=bd6592c37bb2,40087,1732486423713, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0252adfc27b54a76a05585f3f45bf90, server=bd6592c37bb2,40087,1732486423713 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T22:14:01,160 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732486426439.f0252adfc27b54a76a05585f3f45bf90., hostname=bd6592c37bb2,40087,1732486423713, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0252adfc27b54a76a05585f3f45bf90, server=bd6592c37bb2,40087,1732486423713 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T22:14:01,160 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732486426439.f0252adfc27b54a76a05585f3f45bf90., hostname=bd6592c37bb2,40087,1732486423713, seqNum=2 because the exception is null or not the one we care about 2024-11-24T22:14:01,164 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40087 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0252adfc27b54a76a05585f3f45bf90, server=bd6592c37bb2,40087,1732486423713 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-24T22:14:01,165 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40087 {}] ipc.CallRunner(138): callId: 99 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36548 deadline: 1732486501161, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0252adfc27b54a76a05585f3f45bf90, server=bd6592c37bb2,40087,1732486423713 2024-11-24T22:14:01,165 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732486426439.f0252adfc27b54a76a05585f3f45bf90., hostname=bd6592c37bb2,40087,1732486423713, seqNum=2 , the old value is region=TestAcidGuarantees,,1732486426439.f0252adfc27b54a76a05585f3f45bf90., hostname=bd6592c37bb2,40087,1732486423713, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0252adfc27b54a76a05585f3f45bf90, server=bd6592c37bb2,40087,1732486423713 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T22:14:01,165 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732486426439.f0252adfc27b54a76a05585f3f45bf90., hostname=bd6592c37bb2,40087,1732486423713, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0252adfc27b54a76a05585f3f45bf90, server=bd6592c37bb2,40087,1732486423713 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T22:14:01,165 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732486426439.f0252adfc27b54a76a05585f3f45bf90., hostname=bd6592c37bb2,40087,1732486423713, seqNum=2 because the exception is null or not the one we care about 2024-11-24T22:14:01,253 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40087 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0252adfc27b54a76a05585f3f45bf90, server=bd6592c37bb2,40087,1732486423713 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-24T22:14:01,253 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40087 {}] ipc.CallRunner(138): callId: 140 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36532 deadline: 1732486501250, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0252adfc27b54a76a05585f3f45bf90, server=bd6592c37bb2,40087,1732486423713 2024-11-24T22:14:01,254 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732486426439.f0252adfc27b54a76a05585f3f45bf90., hostname=bd6592c37bb2,40087,1732486423713, seqNum=2 , the old value is region=TestAcidGuarantees,,1732486426439.f0252adfc27b54a76a05585f3f45bf90., hostname=bd6592c37bb2,40087,1732486423713, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0252adfc27b54a76a05585f3f45bf90, server=bd6592c37bb2,40087,1732486423713 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T22:14:01,254 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40087 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0252adfc27b54a76a05585f3f45bf90, server=bd6592c37bb2,40087,1732486423713 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-24T22:14:01,254 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732486426439.f0252adfc27b54a76a05585f3f45bf90., hostname=bd6592c37bb2,40087,1732486423713, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0252adfc27b54a76a05585f3f45bf90, server=bd6592c37bb2,40087,1732486423713 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T22:14:01,254 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732486426439.f0252adfc27b54a76a05585f3f45bf90., hostname=bd6592c37bb2,40087,1732486423713, seqNum=2 because the exception is null or not the one we care about 2024-11-24T22:14:01,254 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40087 {}] ipc.CallRunner(138): callId: 169 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36518 deadline: 1732486501251, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0252adfc27b54a76a05585f3f45bf90, server=bd6592c37bb2,40087,1732486423713 2024-11-24T22:14:01,255 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732486426439.f0252adfc27b54a76a05585f3f45bf90., hostname=bd6592c37bb2,40087,1732486423713, seqNum=2 , the old value is region=TestAcidGuarantees,,1732486426439.f0252adfc27b54a76a05585f3f45bf90., hostname=bd6592c37bb2,40087,1732486423713, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0252adfc27b54a76a05585f3f45bf90, server=bd6592c37bb2,40087,1732486423713 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T22:14:01,255 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732486426439.f0252adfc27b54a76a05585f3f45bf90., hostname=bd6592c37bb2,40087,1732486423713, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0252adfc27b54a76a05585f3f45bf90, server=bd6592c37bb2,40087,1732486423713 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T22:14:01,255 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732486426439.f0252adfc27b54a76a05585f3f45bf90., hostname=bd6592c37bb2,40087,1732486423713, seqNum=2 because the exception is null or not the one we care about 2024-11-24T22:14:01,340 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37701 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=13 2024-11-24T22:14:01,341 INFO [RPCClient-NioEventLoopGroup-4-8 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-24T22:14:01,341 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37701 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-24T22:14:01,342 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37701 {}] procedure2.ProcedureExecutor(1139): Stored pid=15, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=15, table=TestAcidGuarantees 2024-11-24T22:14:01,343 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37701 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=15 2024-11-24T22:14:01,343 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=15, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=15, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-24T22:14:01,344 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=15, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=15, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-24T22:14:01,345 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=16, ppid=15, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-24T22:14:01,450 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37701 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=15 2024-11-24T22:14:01,466 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40087 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0252adfc27b54a76a05585f3f45bf90, server=bd6592c37bb2,40087,1732486423713 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-24T22:14:01,466 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40087 {}] ipc.CallRunner(138): callId: 170 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36518 deadline: 1732486501461, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0252adfc27b54a76a05585f3f45bf90, server=bd6592c37bb2,40087,1732486423713 2024-11-24T22:14:01,467 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732486426439.f0252adfc27b54a76a05585f3f45bf90., hostname=bd6592c37bb2,40087,1732486423713, seqNum=2 , the old value is region=TestAcidGuarantees,,1732486426439.f0252adfc27b54a76a05585f3f45bf90., hostname=bd6592c37bb2,40087,1732486423713, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0252adfc27b54a76a05585f3f45bf90, server=bd6592c37bb2,40087,1732486423713 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T22:14:01,467 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732486426439.f0252adfc27b54a76a05585f3f45bf90., hostname=bd6592c37bb2,40087,1732486423713, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0252adfc27b54a76a05585f3f45bf90, server=bd6592c37bb2,40087,1732486423713 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T22:14:01,467 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732486426439.f0252adfc27b54a76a05585f3f45bf90., hostname=bd6592c37bb2,40087,1732486423713, seqNum=2 because the exception is null or not the one we care about 2024-11-24T22:14:01,471 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40087 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0252adfc27b54a76a05585f3f45bf90, server=bd6592c37bb2,40087,1732486423713 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-24T22:14:01,472 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40087 {}] ipc.CallRunner(138): callId: 141 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36532 deadline: 1732486501461, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0252adfc27b54a76a05585f3f45bf90, server=bd6592c37bb2,40087,1732486423713 2024-11-24T22:14:01,473 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732486426439.f0252adfc27b54a76a05585f3f45bf90., hostname=bd6592c37bb2,40087,1732486423713, seqNum=2 , the old value is region=TestAcidGuarantees,,1732486426439.f0252adfc27b54a76a05585f3f45bf90., hostname=bd6592c37bb2,40087,1732486423713, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0252adfc27b54a76a05585f3f45bf90, server=bd6592c37bb2,40087,1732486423713 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T22:14:01,473 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732486426439.f0252adfc27b54a76a05585f3f45bf90., hostname=bd6592c37bb2,40087,1732486423713, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0252adfc27b54a76a05585f3f45bf90, server=bd6592c37bb2,40087,1732486423713 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T22:14:01,473 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732486426439.f0252adfc27b54a76a05585f3f45bf90., hostname=bd6592c37bb2,40087,1732486423713, seqNum=2 because the exception is null or not the one we care about 2024-11-24T22:14:01,506 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=40087 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=16 2024-11-24T22:14:01,506 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bd6592c37bb2:0-1 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732486426439.f0252adfc27b54a76a05585f3f45bf90. 2024-11-24T22:14:01,506 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bd6592c37bb2:0-1 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732486426439.f0252adfc27b54a76a05585f3f45bf90. as already flushing 2024-11-24T22:14:01,506 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bd6592c37bb2:0-1 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732486426439.f0252adfc27b54a76a05585f3f45bf90. 2024-11-24T22:14:01,506 ERROR [RS_FLUSH_OPERATIONS-regionserver/bd6592c37bb2:0-1 {event_type=RS_FLUSH_REGIONS, pid=16}] handler.RSProcedureHandler(58): pid=16 java.io.IOException: Unable to complete flush {ENCODED => f0252adfc27b54a76a05585f3f45bf90, NAME => 'TestAcidGuarantees,,1732486426439.f0252adfc27b54a76a05585f3f45bf90.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T22:14:01,507 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bd6592c37bb2:0-1 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=16 java.io.IOException: Unable to complete flush {ENCODED => f0252adfc27b54a76a05585f3f45bf90, NAME => 'TestAcidGuarantees,,1732486426439.f0252adfc27b54a76a05585f3f45bf90.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T22:14:01,508 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37701 {}] master.HMaster(4177): Remote procedure failed, pid=16 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => f0252adfc27b54a76a05585f3f45bf90, NAME => 'TestAcidGuarantees,,1732486426439.f0252adfc27b54a76a05585f3f45bf90.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => f0252adfc27b54a76a05585f3f45bf90, NAME => 'TestAcidGuarantees,,1732486426439.f0252adfc27b54a76a05585f3f45bf90.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T22:14:01,519 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=26.84 KB at sequenceid=481 (bloomFilter=true), to=hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/f0252adfc27b54a76a05585f3f45bf90/.tmp/A/7160c90e6125430f851ec64126d011ae 2024-11-24T22:14:01,532 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/f0252adfc27b54a76a05585f3f45bf90/.tmp/B/417bf79f9dda4788b24c3865cecc3237 is 50, key is test_row_0/B:col10/1732486440001/Put/seqid=0 2024-11-24T22:14:01,559 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45929 is added to blk_1073741886_1062 (size=12301) 2024-11-24T22:14:01,660 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37701 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=15 2024-11-24T22:14:01,661 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=40087 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=16 2024-11-24T22:14:01,661 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bd6592c37bb2:0-2 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732486426439.f0252adfc27b54a76a05585f3f45bf90. 2024-11-24T22:14:01,662 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bd6592c37bb2:0-2 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732486426439.f0252adfc27b54a76a05585f3f45bf90. as already flushing 2024-11-24T22:14:01,662 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bd6592c37bb2:0-2 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732486426439.f0252adfc27b54a76a05585f3f45bf90. 2024-11-24T22:14:01,662 ERROR [RS_FLUSH_OPERATIONS-regionserver/bd6592c37bb2:0-2 {event_type=RS_FLUSH_REGIONS, pid=16}] handler.RSProcedureHandler(58): pid=16 java.io.IOException: Unable to complete flush {ENCODED => f0252adfc27b54a76a05585f3f45bf90, NAME => 'TestAcidGuarantees,,1732486426439.f0252adfc27b54a76a05585f3f45bf90.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T22:14:01,662 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bd6592c37bb2:0-2 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=16 java.io.IOException: Unable to complete flush {ENCODED => f0252adfc27b54a76a05585f3f45bf90, NAME => 'TestAcidGuarantees,,1732486426439.f0252adfc27b54a76a05585f3f45bf90.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T22:14:01,663 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37701 {}] master.HMaster(4177): Remote procedure failed, pid=16 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => f0252adfc27b54a76a05585f3f45bf90, NAME => 'TestAcidGuarantees,,1732486426439.f0252adfc27b54a76a05585f3f45bf90.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => f0252adfc27b54a76a05585f3f45bf90, NAME => 'TestAcidGuarantees,,1732486426439.f0252adfc27b54a76a05585f3f45bf90.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T22:14:01,772 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40087 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0252adfc27b54a76a05585f3f45bf90, server=bd6592c37bb2,40087,1732486423713 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-24T22:14:01,772 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40087 {}] ipc.CallRunner(138): callId: 171 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36518 deadline: 1732486501770, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0252adfc27b54a76a05585f3f45bf90, server=bd6592c37bb2,40087,1732486423713 2024-11-24T22:14:01,773 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732486426439.f0252adfc27b54a76a05585f3f45bf90., hostname=bd6592c37bb2,40087,1732486423713, seqNum=2 , the old value is region=TestAcidGuarantees,,1732486426439.f0252adfc27b54a76a05585f3f45bf90., hostname=bd6592c37bb2,40087,1732486423713, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0252adfc27b54a76a05585f3f45bf90, server=bd6592c37bb2,40087,1732486423713 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T22:14:01,773 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732486426439.f0252adfc27b54a76a05585f3f45bf90., hostname=bd6592c37bb2,40087,1732486423713, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0252adfc27b54a76a05585f3f45bf90, server=bd6592c37bb2,40087,1732486423713 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T22:14:01,773 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732486426439.f0252adfc27b54a76a05585f3f45bf90., hostname=bd6592c37bb2,40087,1732486423713, seqNum=2 because the exception is null or not the one we care about 2024-11-24T22:14:01,784 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40087 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0252adfc27b54a76a05585f3f45bf90, server=bd6592c37bb2,40087,1732486423713 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-24T22:14:01,784 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40087 {}] ipc.CallRunner(138): callId: 142 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36532 deadline: 1732486501781, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0252adfc27b54a76a05585f3f45bf90, server=bd6592c37bb2,40087,1732486423713 2024-11-24T22:14:01,784 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732486426439.f0252adfc27b54a76a05585f3f45bf90., hostname=bd6592c37bb2,40087,1732486423713, seqNum=2 , the old value is region=TestAcidGuarantees,,1732486426439.f0252adfc27b54a76a05585f3f45bf90., hostname=bd6592c37bb2,40087,1732486423713, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0252adfc27b54a76a05585f3f45bf90, server=bd6592c37bb2,40087,1732486423713 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T22:14:01,784 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732486426439.f0252adfc27b54a76a05585f3f45bf90., hostname=bd6592c37bb2,40087,1732486423713, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0252adfc27b54a76a05585f3f45bf90, server=bd6592c37bb2,40087,1732486423713 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T22:14:01,784 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732486426439.f0252adfc27b54a76a05585f3f45bf90., hostname=bd6592c37bb2,40087,1732486423713, seqNum=2 because the exception is null or not the one we care about 2024-11-24T22:14:01,815 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=40087 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=16 2024-11-24T22:14:01,815 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bd6592c37bb2:0-0 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732486426439.f0252adfc27b54a76a05585f3f45bf90. 2024-11-24T22:14:01,816 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bd6592c37bb2:0-0 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732486426439.f0252adfc27b54a76a05585f3f45bf90. as already flushing 2024-11-24T22:14:01,816 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bd6592c37bb2:0-0 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732486426439.f0252adfc27b54a76a05585f3f45bf90. 2024-11-24T22:14:01,816 ERROR [RS_FLUSH_OPERATIONS-regionserver/bd6592c37bb2:0-0 {event_type=RS_FLUSH_REGIONS, pid=16}] handler.RSProcedureHandler(58): pid=16 java.io.IOException: Unable to complete flush {ENCODED => f0252adfc27b54a76a05585f3f45bf90, NAME => 'TestAcidGuarantees,,1732486426439.f0252adfc27b54a76a05585f3f45bf90.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T22:14:01,816 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bd6592c37bb2:0-0 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=16 java.io.IOException: Unable to complete flush {ENCODED => f0252adfc27b54a76a05585f3f45bf90, NAME => 'TestAcidGuarantees,,1732486426439.f0252adfc27b54a76a05585f3f45bf90.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T22:14:01,817 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37701 {}] master.HMaster(4177): Remote procedure failed, pid=16 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => f0252adfc27b54a76a05585f3f45bf90, NAME => 'TestAcidGuarantees,,1732486426439.f0252adfc27b54a76a05585f3f45bf90.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => f0252adfc27b54a76a05585f3f45bf90, NAME => 'TestAcidGuarantees,,1732486426439.f0252adfc27b54a76a05585f3f45bf90.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T22:14:01,960 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=26.84 KB at sequenceid=481 (bloomFilter=true), to=hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/f0252adfc27b54a76a05585f3f45bf90/.tmp/B/417bf79f9dda4788b24c3865cecc3237 2024-11-24T22:14:01,969 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=40087 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=16 2024-11-24T22:14:01,970 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bd6592c37bb2:0-1 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732486426439.f0252adfc27b54a76a05585f3f45bf90. 2024-11-24T22:14:01,970 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bd6592c37bb2:0-1 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732486426439.f0252adfc27b54a76a05585f3f45bf90. as already flushing 2024-11-24T22:14:01,970 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bd6592c37bb2:0-1 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732486426439.f0252adfc27b54a76a05585f3f45bf90. 2024-11-24T22:14:01,970 ERROR [RS_FLUSH_OPERATIONS-regionserver/bd6592c37bb2:0-1 {event_type=RS_FLUSH_REGIONS, pid=16}] handler.RSProcedureHandler(58): pid=16 java.io.IOException: Unable to complete flush {ENCODED => f0252adfc27b54a76a05585f3f45bf90, NAME => 'TestAcidGuarantees,,1732486426439.f0252adfc27b54a76a05585f3f45bf90.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T22:14:01,970 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bd6592c37bb2:0-1 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=16 java.io.IOException: Unable to complete flush {ENCODED => f0252adfc27b54a76a05585f3f45bf90, NAME => 'TestAcidGuarantees,,1732486426439.f0252adfc27b54a76a05585f3f45bf90.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T22:14:01,970 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37701 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=15 2024-11-24T22:14:01,971 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/f0252adfc27b54a76a05585f3f45bf90/.tmp/C/6b841508335b4fd0b3e088cb97a7ba51 is 50, key is test_row_0/C:col10/1732486440001/Put/seqid=0 2024-11-24T22:14:01,971 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37701 {}] master.HMaster(4177): Remote procedure failed, pid=16 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => f0252adfc27b54a76a05585f3f45bf90, NAME => 'TestAcidGuarantees,,1732486426439.f0252adfc27b54a76a05585f3f45bf90.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => f0252adfc27b54a76a05585f3f45bf90, NAME => 'TestAcidGuarantees,,1732486426439.f0252adfc27b54a76a05585f3f45bf90.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T22:14:01,977 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45929 is added to blk_1073741887_1063 (size=12301) 2024-11-24T22:14:02,124 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=40087 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=16 2024-11-24T22:14:02,124 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bd6592c37bb2:0-2 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732486426439.f0252adfc27b54a76a05585f3f45bf90. 2024-11-24T22:14:02,124 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bd6592c37bb2:0-2 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732486426439.f0252adfc27b54a76a05585f3f45bf90. as already flushing 2024-11-24T22:14:02,124 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bd6592c37bb2:0-2 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732486426439.f0252adfc27b54a76a05585f3f45bf90. 2024-11-24T22:14:02,124 ERROR [RS_FLUSH_OPERATIONS-regionserver/bd6592c37bb2:0-2 {event_type=RS_FLUSH_REGIONS, pid=16}] handler.RSProcedureHandler(58): pid=16 java.io.IOException: Unable to complete flush {ENCODED => f0252adfc27b54a76a05585f3f45bf90, NAME => 'TestAcidGuarantees,,1732486426439.f0252adfc27b54a76a05585f3f45bf90.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T22:14:02,125 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bd6592c37bb2:0-2 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=16 java.io.IOException: Unable to complete flush {ENCODED => f0252adfc27b54a76a05585f3f45bf90, NAME => 'TestAcidGuarantees,,1732486426439.f0252adfc27b54a76a05585f3f45bf90.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T22:14:02,125 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37701 {}] master.HMaster(4177): Remote procedure failed, pid=16 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => f0252adfc27b54a76a05585f3f45bf90, NAME => 'TestAcidGuarantees,,1732486426439.f0252adfc27b54a76a05585f3f45bf90.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => f0252adfc27b54a76a05585f3f45bf90, NAME => 'TestAcidGuarantees,,1732486426439.f0252adfc27b54a76a05585f3f45bf90.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T22:14:02,163 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40087 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0252adfc27b54a76a05585f3f45bf90, server=bd6592c37bb2,40087,1732486423713 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-24T22:14:02,163 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40087 {}] ipc.CallRunner(138): callId: 106 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36590 deadline: 1732486502161, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0252adfc27b54a76a05585f3f45bf90, server=bd6592c37bb2,40087,1732486423713 2024-11-24T22:14:02,169 WARN [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRpcRetryingCaller(177): Call to bd6592c37bb2,40087,1732486423713 for 'test_row_1' in f0252adfc27b54a76a05585f3f45bf90 of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2167 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0252adfc27b54a76a05585f3f45bf90, server=bd6592c37bb2,40087,1732486423713 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor44.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0252adfc27b54a76a05585f3f45bf90, server=bd6592c37bb2,40087,1732486423713 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-24T22:14:02,170 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732486426439.f0252adfc27b54a76a05585f3f45bf90., hostname=bd6592c37bb2,40087,1732486423713, seqNum=2 , the old value is region=TestAcidGuarantees,,1732486426439.f0252adfc27b54a76a05585f3f45bf90., hostname=bd6592c37bb2,40087,1732486423713, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0252adfc27b54a76a05585f3f45bf90, server=bd6592c37bb2,40087,1732486423713 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T22:14:02,170 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732486426439.f0252adfc27b54a76a05585f3f45bf90., hostname=bd6592c37bb2,40087,1732486423713, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0252adfc27b54a76a05585f3f45bf90, server=bd6592c37bb2,40087,1732486423713 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T22:14:02,170 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732486426439.f0252adfc27b54a76a05585f3f45bf90., hostname=bd6592c37bb2,40087,1732486423713, seqNum=2 because the exception is null or not the one we care about 2024-11-24T22:14:02,179 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40087 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0252adfc27b54a76a05585f3f45bf90, server=bd6592c37bb2,40087,1732486423713 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-24T22:14:02,179 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40087 {}] ipc.CallRunner(138): callId: 119 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36562 deadline: 1732486502170, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0252adfc27b54a76a05585f3f45bf90, server=bd6592c37bb2,40087,1732486423713 2024-11-24T22:14:02,180 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40087 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0252adfc27b54a76a05585f3f45bf90, server=bd6592c37bb2,40087,1732486423713 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-24T22:14:02,180 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40087 {}] ipc.CallRunner(138): callId: 100 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36548 deadline: 1732486502171, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0252adfc27b54a76a05585f3f45bf90, server=bd6592c37bb2,40087,1732486423713 2024-11-24T22:14:02,181 WARN [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRpcRetryingCaller(177): Call to bd6592c37bb2,40087,1732486423713 for 'test_row_0' in f0252adfc27b54a76a05585f3f45bf90 of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2176 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0252adfc27b54a76a05585f3f45bf90, server=bd6592c37bb2,40087,1732486423713 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor44.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0252adfc27b54a76a05585f3f45bf90, server=bd6592c37bb2,40087,1732486423713 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-24T22:14:02,181 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732486426439.f0252adfc27b54a76a05585f3f45bf90., hostname=bd6592c37bb2,40087,1732486423713, seqNum=2 , the old value is region=TestAcidGuarantees,,1732486426439.f0252adfc27b54a76a05585f3f45bf90., hostname=bd6592c37bb2,40087,1732486423713, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0252adfc27b54a76a05585f3f45bf90, server=bd6592c37bb2,40087,1732486423713 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T22:14:02,181 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732486426439.f0252adfc27b54a76a05585f3f45bf90., hostname=bd6592c37bb2,40087,1732486423713, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0252adfc27b54a76a05585f3f45bf90, server=bd6592c37bb2,40087,1732486423713 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T22:14:02,181 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732486426439.f0252adfc27b54a76a05585f3f45bf90., hostname=bd6592c37bb2,40087,1732486423713, seqNum=2 because the exception is null or not the one we care about 2024-11-24T22:14:02,182 WARN [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRpcRetryingCaller(177): Call to bd6592c37bb2,40087,1732486423713 for 'test_row_0' in f0252adfc27b54a76a05585f3f45bf90 of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2178 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0252adfc27b54a76a05585f3f45bf90, server=bd6592c37bb2,40087,1732486423713 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor44.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0252adfc27b54a76a05585f3f45bf90, server=bd6592c37bb2,40087,1732486423713 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-24T22:14:02,182 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732486426439.f0252adfc27b54a76a05585f3f45bf90., hostname=bd6592c37bb2,40087,1732486423713, seqNum=2 , the old value is region=TestAcidGuarantees,,1732486426439.f0252adfc27b54a76a05585f3f45bf90., hostname=bd6592c37bb2,40087,1732486423713, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0252adfc27b54a76a05585f3f45bf90, server=bd6592c37bb2,40087,1732486423713 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T22:14:02,183 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732486426439.f0252adfc27b54a76a05585f3f45bf90., hostname=bd6592c37bb2,40087,1732486423713, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0252adfc27b54a76a05585f3f45bf90, server=bd6592c37bb2,40087,1732486423713 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T22:14:02,183 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732486426439.f0252adfc27b54a76a05585f3f45bf90., hostname=bd6592c37bb2,40087,1732486423713, seqNum=2 because the exception is null or not the one we care about 2024-11-24T22:14:02,278 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=40087 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=16 2024-11-24T22:14:02,278 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bd6592c37bb2:0-0 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732486426439.f0252adfc27b54a76a05585f3f45bf90. 2024-11-24T22:14:02,279 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bd6592c37bb2:0-0 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732486426439.f0252adfc27b54a76a05585f3f45bf90. as already flushing 2024-11-24T22:14:02,279 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bd6592c37bb2:0-0 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732486426439.f0252adfc27b54a76a05585f3f45bf90. 2024-11-24T22:14:02,279 ERROR [RS_FLUSH_OPERATIONS-regionserver/bd6592c37bb2:0-0 {event_type=RS_FLUSH_REGIONS, pid=16}] handler.RSProcedureHandler(58): pid=16 java.io.IOException: Unable to complete flush {ENCODED => f0252adfc27b54a76a05585f3f45bf90, NAME => 'TestAcidGuarantees,,1732486426439.f0252adfc27b54a76a05585f3f45bf90.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T22:14:02,279 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bd6592c37bb2:0-0 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=16 java.io.IOException: Unable to complete flush {ENCODED => f0252adfc27b54a76a05585f3f45bf90, NAME => 'TestAcidGuarantees,,1732486426439.f0252adfc27b54a76a05585f3f45bf90.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T22:14:02,280 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37701 {}] master.HMaster(4177): Remote procedure failed, pid=16 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => f0252adfc27b54a76a05585f3f45bf90, NAME => 'TestAcidGuarantees,,1732486426439.f0252adfc27b54a76a05585f3f45bf90.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => f0252adfc27b54a76a05585f3f45bf90, NAME => 'TestAcidGuarantees,,1732486426439.f0252adfc27b54a76a05585f3f45bf90.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T22:14:02,292 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40087 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0252adfc27b54a76a05585f3f45bf90, server=bd6592c37bb2,40087,1732486423713 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-24T22:14:02,292 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40087 {}] ipc.CallRunner(138): callId: 172 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36518 deadline: 1732486502281, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0252adfc27b54a76a05585f3f45bf90, server=bd6592c37bb2,40087,1732486423713 2024-11-24T22:14:02,292 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40087 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0252adfc27b54a76a05585f3f45bf90, server=bd6592c37bb2,40087,1732486423713 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-24T22:14:02,293 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40087 {}] ipc.CallRunner(138): callId: 143 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36532 deadline: 1732486502291, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0252adfc27b54a76a05585f3f45bf90, server=bd6592c37bb2,40087,1732486423713 2024-11-24T22:14:02,293 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732486426439.f0252adfc27b54a76a05585f3f45bf90., hostname=bd6592c37bb2,40087,1732486423713, seqNum=2 , the old value is region=TestAcidGuarantees,,1732486426439.f0252adfc27b54a76a05585f3f45bf90., hostname=bd6592c37bb2,40087,1732486423713, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0252adfc27b54a76a05585f3f45bf90, server=bd6592c37bb2,40087,1732486423713 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T22:14:02,293 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732486426439.f0252adfc27b54a76a05585f3f45bf90., hostname=bd6592c37bb2,40087,1732486423713, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0252adfc27b54a76a05585f3f45bf90, server=bd6592c37bb2,40087,1732486423713 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T22:14:02,293 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732486426439.f0252adfc27b54a76a05585f3f45bf90., hostname=bd6592c37bb2,40087,1732486423713, seqNum=2 because the exception is null or not the one we care about 2024-11-24T22:14:02,293 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732486426439.f0252adfc27b54a76a05585f3f45bf90., hostname=bd6592c37bb2,40087,1732486423713, seqNum=2 , the old value is region=TestAcidGuarantees,,1732486426439.f0252adfc27b54a76a05585f3f45bf90., hostname=bd6592c37bb2,40087,1732486423713, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0252adfc27b54a76a05585f3f45bf90, server=bd6592c37bb2,40087,1732486423713 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T22:14:02,294 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732486426439.f0252adfc27b54a76a05585f3f45bf90., hostname=bd6592c37bb2,40087,1732486423713, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0252adfc27b54a76a05585f3f45bf90, server=bd6592c37bb2,40087,1732486423713 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T22:14:02,294 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732486426439.f0252adfc27b54a76a05585f3f45bf90., hostname=bd6592c37bb2,40087,1732486423713, seqNum=2 because the exception is null or not the one we care about 2024-11-24T22:14:02,379 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=26.84 KB at sequenceid=481 (bloomFilter=true), to=hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/f0252adfc27b54a76a05585f3f45bf90/.tmp/C/6b841508335b4fd0b3e088cb97a7ba51 2024-11-24T22:14:02,388 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/f0252adfc27b54a76a05585f3f45bf90/.tmp/A/7160c90e6125430f851ec64126d011ae as hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/f0252adfc27b54a76a05585f3f45bf90/A/7160c90e6125430f851ec64126d011ae 2024-11-24T22:14:02,395 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/f0252adfc27b54a76a05585f3f45bf90/A/7160c90e6125430f851ec64126d011ae, entries=200, sequenceid=481, filesize=14.4 K 2024-11-24T22:14:02,398 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/f0252adfc27b54a76a05585f3f45bf90/.tmp/B/417bf79f9dda4788b24c3865cecc3237 as hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/f0252adfc27b54a76a05585f3f45bf90/B/417bf79f9dda4788b24c3865cecc3237 2024-11-24T22:14:02,425 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/f0252adfc27b54a76a05585f3f45bf90/B/417bf79f9dda4788b24c3865cecc3237, entries=150, sequenceid=481, filesize=12.0 K 2024-11-24T22:14:02,426 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/f0252adfc27b54a76a05585f3f45bf90/.tmp/C/6b841508335b4fd0b3e088cb97a7ba51 as hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/f0252adfc27b54a76a05585f3f45bf90/C/6b841508335b4fd0b3e088cb97a7ba51 2024-11-24T22:14:02,432 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=40087 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=16 2024-11-24T22:14:02,433 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bd6592c37bb2:0-1 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732486426439.f0252adfc27b54a76a05585f3f45bf90. 2024-11-24T22:14:02,433 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bd6592c37bb2:0-1 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732486426439.f0252adfc27b54a76a05585f3f45bf90. as already flushing 2024-11-24T22:14:02,433 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bd6592c37bb2:0-1 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732486426439.f0252adfc27b54a76a05585f3f45bf90. 2024-11-24T22:14:02,433 ERROR [RS_FLUSH_OPERATIONS-regionserver/bd6592c37bb2:0-1 {event_type=RS_FLUSH_REGIONS, pid=16}] handler.RSProcedureHandler(58): pid=16 java.io.IOException: Unable to complete flush {ENCODED => f0252adfc27b54a76a05585f3f45bf90, NAME => 'TestAcidGuarantees,,1732486426439.f0252adfc27b54a76a05585f3f45bf90.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T22:14:02,433 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bd6592c37bb2:0-1 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=16 java.io.IOException: Unable to complete flush {ENCODED => f0252adfc27b54a76a05585f3f45bf90, NAME => 'TestAcidGuarantees,,1732486426439.f0252adfc27b54a76a05585f3f45bf90.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T22:14:02,435 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37701 {}] master.HMaster(4177): Remote procedure failed, pid=16 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => f0252adfc27b54a76a05585f3f45bf90, NAME => 'TestAcidGuarantees,,1732486426439.f0252adfc27b54a76a05585f3f45bf90.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => f0252adfc27b54a76a05585f3f45bf90, NAME => 'TestAcidGuarantees,,1732486426439.f0252adfc27b54a76a05585f3f45bf90.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T22:14:02,436 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/f0252adfc27b54a76a05585f3f45bf90/C/6b841508335b4fd0b3e088cb97a7ba51, entries=150, sequenceid=481, filesize=12.0 K 2024-11-24T22:14:02,439 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~80.51 KB/82440, heapSize ~239.77 KB/245520, currentSize=100.63 KB/103050 for f0252adfc27b54a76a05585f3f45bf90 in 1353ms, sequenceid=481, compaction requested=true 2024-11-24T22:14:02,439 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for f0252adfc27b54a76a05585f3f45bf90: 2024-11-24T22:14:02,439 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store f0252adfc27b54a76a05585f3f45bf90:A, priority=-2147483648, current under compaction store size is 1 2024-11-24T22:14:02,439 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-24T22:14:02,439 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store f0252adfc27b54a76a05585f3f45bf90:B, priority=-2147483648, current under compaction store size is 2 2024-11-24T22:14:02,439 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=2), splitQueue=0 2024-11-24T22:14:02,439 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store f0252adfc27b54a76a05585f3f45bf90:C, priority=-2147483648, current under compaction store size is 3 2024-11-24T22:14:02,439 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=3), splitQueue=0 2024-11-24T22:14:02,439 DEBUG [RS:0;bd6592c37bb2:40087-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-24T22:14:02,441 DEBUG [RS:0;bd6592c37bb2:40087-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-24T22:14:02,441 DEBUG [RS:0;bd6592c37bb2:40087-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 52088 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-24T22:14:02,441 DEBUG [RS:0;bd6592c37bb2:40087-shortCompactions-0 {}] regionserver.HStore(1541): f0252adfc27b54a76a05585f3f45bf90/A is initiating minor compaction (all files) 2024-11-24T22:14:02,442 INFO [RS:0;bd6592c37bb2:40087-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of f0252adfc27b54a76a05585f3f45bf90/A in TestAcidGuarantees,,1732486426439.f0252adfc27b54a76a05585f3f45bf90. 2024-11-24T22:14:02,442 INFO [RS:0;bd6592c37bb2:40087-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/f0252adfc27b54a76a05585f3f45bf90/A/c2bd8d33a4064e3b8a2f0df67e949fa7, hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/f0252adfc27b54a76a05585f3f45bf90/A/858d340a75944cf79e54a6945720ed2e, hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/f0252adfc27b54a76a05585f3f45bf90/A/5bcdeaa73e0f49c29d59c30322b2c997, hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/f0252adfc27b54a76a05585f3f45bf90/A/7160c90e6125430f851ec64126d011ae] into tmpdir=hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/f0252adfc27b54a76a05585f3f45bf90/.tmp, totalSize=50.9 K 2024-11-24T22:14:02,444 DEBUG [RS:0;bd6592c37bb2:40087-shortCompactions-0 {}] compactions.Compactor(225): Compacting c2bd8d33a4064e3b8a2f0df67e949fa7, keycount=150, bloomtype=ROW, size=12.4 K, encoding=NONE, compression=NONE, seqNum=431, earliestPutTs=1732486436617 2024-11-24T22:14:02,444 DEBUG [RS:0;bd6592c37bb2:40087-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 49648 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-24T22:14:02,444 DEBUG [RS:0;bd6592c37bb2:40087-longCompactions-0 {}] regionserver.HStore(1541): f0252adfc27b54a76a05585f3f45bf90/C is initiating minor compaction (all files) 2024-11-24T22:14:02,444 INFO [RS:0;bd6592c37bb2:40087-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of f0252adfc27b54a76a05585f3f45bf90/C in TestAcidGuarantees,,1732486426439.f0252adfc27b54a76a05585f3f45bf90. 2024-11-24T22:14:02,445 INFO [RS:0;bd6592c37bb2:40087-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/f0252adfc27b54a76a05585f3f45bf90/C/2eeb43633aa74ec58c0313ac4a1fc08b, hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/f0252adfc27b54a76a05585f3f45bf90/C/e8d52e0fc5344af581b6bdc556bace1e, hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/f0252adfc27b54a76a05585f3f45bf90/C/d1accac28c8541a0b967ecefbff6faa6, hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/f0252adfc27b54a76a05585f3f45bf90/C/6b841508335b4fd0b3e088cb97a7ba51] into tmpdir=hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/f0252adfc27b54a76a05585f3f45bf90/.tmp, totalSize=48.5 K 2024-11-24T22:14:02,445 DEBUG [RS:0;bd6592c37bb2:40087-shortCompactions-0 {}] compactions.Compactor(225): Compacting 858d340a75944cf79e54a6945720ed2e, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=445, earliestPutTs=1732486437795 2024-11-24T22:14:02,445 DEBUG [RS:0;bd6592c37bb2:40087-longCompactions-0 {}] compactions.Compactor(225): Compacting 2eeb43633aa74ec58c0313ac4a1fc08b, keycount=150, bloomtype=ROW, size=12.4 K, encoding=NONE, compression=NONE, seqNum=431, earliestPutTs=1732486436617 2024-11-24T22:14:02,445 DEBUG [RS:0;bd6592c37bb2:40087-shortCompactions-0 {}] compactions.Compactor(225): Compacting 5bcdeaa73e0f49c29d59c30322b2c997, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=466, earliestPutTs=1732486438921 2024-11-24T22:14:02,446 DEBUG [RS:0;bd6592c37bb2:40087-longCompactions-0 {}] compactions.Compactor(225): Compacting e8d52e0fc5344af581b6bdc556bace1e, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=445, earliestPutTs=1732486437795 2024-11-24T22:14:02,446 DEBUG [RS:0;bd6592c37bb2:40087-shortCompactions-0 {}] compactions.Compactor(225): Compacting 7160c90e6125430f851ec64126d011ae, keycount=200, bloomtype=ROW, size=14.4 K, encoding=NONE, compression=NONE, seqNum=481, earliestPutTs=1732486439980 2024-11-24T22:14:02,447 DEBUG [RS:0;bd6592c37bb2:40087-longCompactions-0 {}] compactions.Compactor(225): Compacting d1accac28c8541a0b967ecefbff6faa6, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=466, earliestPutTs=1732486438921 2024-11-24T22:14:02,447 DEBUG [RS:0;bd6592c37bb2:40087-longCompactions-0 {}] compactions.Compactor(225): Compacting 6b841508335b4fd0b3e088cb97a7ba51, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=481, earliestPutTs=1732486439980 2024-11-24T22:14:02,462 INFO [RS:0;bd6592c37bb2:40087-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): f0252adfc27b54a76a05585f3f45bf90#C#compaction#51 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-24T22:14:02,462 INFO [RS:0;bd6592c37bb2:40087-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): f0252adfc27b54a76a05585f3f45bf90#A#compaction#52 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-24T22:14:02,462 DEBUG [RS:0;bd6592c37bb2:40087-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/f0252adfc27b54a76a05585f3f45bf90/.tmp/A/1dc58facd3964ecdb9a0f4fffee4cfe6 is 50, key is test_row_0/A:col10/1732486440001/Put/seqid=0 2024-11-24T22:14:02,462 DEBUG [RS:0;bd6592c37bb2:40087-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/f0252adfc27b54a76a05585f3f45bf90/.tmp/C/984a805e6d2a48d3b398a98cf43f97d2 is 50, key is test_row_0/C:col10/1732486440001/Put/seqid=0 2024-11-24T22:14:02,476 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45929 is added to blk_1073741888_1064 (size=12881) 2024-11-24T22:14:02,481 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37701 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=15 2024-11-24T22:14:02,486 DEBUG [RS:0;bd6592c37bb2:40087-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/f0252adfc27b54a76a05585f3f45bf90/.tmp/C/984a805e6d2a48d3b398a98cf43f97d2 as hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/f0252adfc27b54a76a05585f3f45bf90/C/984a805e6d2a48d3b398a98cf43f97d2 2024-11-24T22:14:02,493 INFO [RS:0;bd6592c37bb2:40087-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in f0252adfc27b54a76a05585f3f45bf90/C of f0252adfc27b54a76a05585f3f45bf90 into 984a805e6d2a48d3b398a98cf43f97d2(size=12.6 K), total size for store is 12.6 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-24T22:14:02,493 DEBUG [RS:0;bd6592c37bb2:40087-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for f0252adfc27b54a76a05585f3f45bf90: 2024-11-24T22:14:02,494 INFO [RS:0;bd6592c37bb2:40087-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732486426439.f0252adfc27b54a76a05585f3f45bf90., storeName=f0252adfc27b54a76a05585f3f45bf90/C, priority=12, startTime=1732486442439; duration=0sec 2024-11-24T22:14:02,494 DEBUG [RS:0;bd6592c37bb2:40087-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-24T22:14:02,494 DEBUG [RS:0;bd6592c37bb2:40087-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: f0252adfc27b54a76a05585f3f45bf90:C 2024-11-24T22:14:02,494 DEBUG [RS:0;bd6592c37bb2:40087-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-24T22:14:02,496 DEBUG [RS:0;bd6592c37bb2:40087-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 49648 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-24T22:14:02,496 DEBUG [RS:0;bd6592c37bb2:40087-longCompactions-0 {}] regionserver.HStore(1541): f0252adfc27b54a76a05585f3f45bf90/B is initiating minor compaction (all files) 2024-11-24T22:14:02,496 INFO [RS:0;bd6592c37bb2:40087-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of f0252adfc27b54a76a05585f3f45bf90/B in TestAcidGuarantees,,1732486426439.f0252adfc27b54a76a05585f3f45bf90. 2024-11-24T22:14:02,496 INFO [RS:0;bd6592c37bb2:40087-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/f0252adfc27b54a76a05585f3f45bf90/B/9e9ff2c611584faf92b0cdcb1129728e, hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/f0252adfc27b54a76a05585f3f45bf90/B/a57df684ebd4407e9ca9d68f74a40615, hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/f0252adfc27b54a76a05585f3f45bf90/B/56630cf0374945f290b0ada498ece426, hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/f0252adfc27b54a76a05585f3f45bf90/B/417bf79f9dda4788b24c3865cecc3237] into tmpdir=hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/f0252adfc27b54a76a05585f3f45bf90/.tmp, totalSize=48.5 K 2024-11-24T22:14:02,497 DEBUG [RS:0;bd6592c37bb2:40087-longCompactions-0 {}] compactions.Compactor(225): Compacting 9e9ff2c611584faf92b0cdcb1129728e, keycount=150, bloomtype=ROW, size=12.4 K, encoding=NONE, compression=NONE, seqNum=431, earliestPutTs=1732486436617 2024-11-24T22:14:02,498 DEBUG [RS:0;bd6592c37bb2:40087-longCompactions-0 {}] compactions.Compactor(225): Compacting a57df684ebd4407e9ca9d68f74a40615, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=445, earliestPutTs=1732486437795 2024-11-24T22:14:02,498 DEBUG [RS:0;bd6592c37bb2:40087-longCompactions-0 {}] compactions.Compactor(225): Compacting 56630cf0374945f290b0ada498ece426, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=466, earliestPutTs=1732486438921 2024-11-24T22:14:02,498 DEBUG [RS:0;bd6592c37bb2:40087-longCompactions-0 {}] compactions.Compactor(225): Compacting 417bf79f9dda4788b24c3865cecc3237, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=481, earliestPutTs=1732486439980 2024-11-24T22:14:02,504 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45929 is added to blk_1073741889_1065 (size=12881) 2024-11-24T22:14:02,513 DEBUG [RS:0;bd6592c37bb2:40087-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/f0252adfc27b54a76a05585f3f45bf90/.tmp/A/1dc58facd3964ecdb9a0f4fffee4cfe6 as hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/f0252adfc27b54a76a05585f3f45bf90/A/1dc58facd3964ecdb9a0f4fffee4cfe6 2024-11-24T22:14:02,521 INFO [RS:0;bd6592c37bb2:40087-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): f0252adfc27b54a76a05585f3f45bf90#B#compaction#53 average throughput is 0.73 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-24T22:14:02,522 INFO [RS:0;bd6592c37bb2:40087-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in f0252adfc27b54a76a05585f3f45bf90/A of f0252adfc27b54a76a05585f3f45bf90 into 1dc58facd3964ecdb9a0f4fffee4cfe6(size=12.6 K), total size for store is 12.6 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-24T22:14:02,522 DEBUG [RS:0;bd6592c37bb2:40087-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for f0252adfc27b54a76a05585f3f45bf90: 2024-11-24T22:14:02,522 INFO [RS:0;bd6592c37bb2:40087-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732486426439.f0252adfc27b54a76a05585f3f45bf90., storeName=f0252adfc27b54a76a05585f3f45bf90/A, priority=12, startTime=1732486442439; duration=0sec 2024-11-24T22:14:02,522 DEBUG [RS:0;bd6592c37bb2:40087-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/f0252adfc27b54a76a05585f3f45bf90/.tmp/B/f786dc1420844a9da33ca12b1b422ea3 is 50, key is test_row_0/B:col10/1732486440001/Put/seqid=0 2024-11-24T22:14:02,523 DEBUG [RS:0;bd6592c37bb2:40087-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-24T22:14:02,523 DEBUG [RS:0;bd6592c37bb2:40087-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: f0252adfc27b54a76a05585f3f45bf90:A 2024-11-24T22:14:02,552 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45929 is added to blk_1073741890_1066 (size=12881) 2024-11-24T22:14:02,590 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=40087 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=16 2024-11-24T22:14:02,590 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bd6592c37bb2:0-2 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732486426439.f0252adfc27b54a76a05585f3f45bf90. 2024-11-24T22:14:02,591 INFO [RS_FLUSH_OPERATIONS-regionserver/bd6592c37bb2:0-2 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.HRegion(2902): Flushing f0252adfc27b54a76a05585f3f45bf90 3/3 column families, dataSize=100.63 KB heapSize=299.58 KB 2024-11-24T22:14:02,591 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bd6592c37bb2:0-2 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.CompactingMemStore(205): FLUSHING TO DISK f0252adfc27b54a76a05585f3f45bf90, store=A 2024-11-24T22:14:02,591 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bd6592c37bb2:0-2 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-24T22:14:02,591 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bd6592c37bb2:0-2 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.CompactingMemStore(205): FLUSHING TO DISK f0252adfc27b54a76a05585f3f45bf90, store=B 2024-11-24T22:14:02,591 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bd6592c37bb2:0-2 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-24T22:14:02,591 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bd6592c37bb2:0-2 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.CompactingMemStore(205): FLUSHING TO DISK f0252adfc27b54a76a05585f3f45bf90, store=C 2024-11-24T22:14:02,591 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bd6592c37bb2:0-2 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-24T22:14:02,623 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bd6592c37bb2:0-2 {event_type=RS_FLUSH_REGIONS, pid=16}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/f0252adfc27b54a76a05585f3f45bf90/.tmp/A/f4702fb2e5c54e189b3430b90cd93ef8 is 50, key is test_row_0/A:col10/1732486441146/Put/seqid=0 2024-11-24T22:14:02,634 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45929 is added to blk_1073741891_1067 (size=12301) 2024-11-24T22:14:02,637 INFO [RS_FLUSH_OPERATIONS-regionserver/bd6592c37bb2:0-2 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=33.54 KB at sequenceid=501 (bloomFilter=true), to=hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/f0252adfc27b54a76a05585f3f45bf90/.tmp/A/f4702fb2e5c54e189b3430b90cd93ef8 2024-11-24T22:14:02,649 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bd6592c37bb2:0-2 {event_type=RS_FLUSH_REGIONS, pid=16}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/f0252adfc27b54a76a05585f3f45bf90/.tmp/B/7ddf65b17c1948cb8dac4dce2ff012c6 is 50, key is test_row_0/B:col10/1732486441146/Put/seqid=0 2024-11-24T22:14:02,661 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45929 is added to blk_1073741892_1068 (size=12301) 2024-11-24T22:14:02,962 DEBUG [RS:0;bd6592c37bb2:40087-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/f0252adfc27b54a76a05585f3f45bf90/.tmp/B/f786dc1420844a9da33ca12b1b422ea3 as hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/f0252adfc27b54a76a05585f3f45bf90/B/f786dc1420844a9da33ca12b1b422ea3 2024-11-24T22:14:02,972 INFO [RS:0;bd6592c37bb2:40087-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in f0252adfc27b54a76a05585f3f45bf90/B of f0252adfc27b54a76a05585f3f45bf90 into f786dc1420844a9da33ca12b1b422ea3(size=12.6 K), total size for store is 12.6 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-24T22:14:02,972 DEBUG [RS:0;bd6592c37bb2:40087-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for f0252adfc27b54a76a05585f3f45bf90: 2024-11-24T22:14:02,972 INFO [RS:0;bd6592c37bb2:40087-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732486426439.f0252adfc27b54a76a05585f3f45bf90., storeName=f0252adfc27b54a76a05585f3f45bf90/B, priority=12, startTime=1732486442439; duration=0sec 2024-11-24T22:14:02,972 DEBUG [RS:0;bd6592c37bb2:40087-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-24T22:14:02,972 DEBUG [RS:0;bd6592c37bb2:40087-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: f0252adfc27b54a76a05585f3f45bf90:B 2024-11-24T22:14:03,062 INFO [RS_FLUSH_OPERATIONS-regionserver/bd6592c37bb2:0-2 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=33.54 KB at sequenceid=501 (bloomFilter=true), to=hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/f0252adfc27b54a76a05585f3f45bf90/.tmp/B/7ddf65b17c1948cb8dac4dce2ff012c6 2024-11-24T22:14:03,074 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bd6592c37bb2:0-2 {event_type=RS_FLUSH_REGIONS, pid=16}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/f0252adfc27b54a76a05585f3f45bf90/.tmp/C/fc8735a74d5548058f328073367c7532 is 50, key is test_row_0/C:col10/1732486441146/Put/seqid=0 2024-11-24T22:14:03,083 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45929 is added to blk_1073741893_1069 (size=12301) 2024-11-24T22:14:03,084 INFO [RS_FLUSH_OPERATIONS-regionserver/bd6592c37bb2:0-2 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=33.54 KB at sequenceid=501 (bloomFilter=true), to=hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/f0252adfc27b54a76a05585f3f45bf90/.tmp/C/fc8735a74d5548058f328073367c7532 2024-11-24T22:14:03,091 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bd6592c37bb2:0-2 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/f0252adfc27b54a76a05585f3f45bf90/.tmp/A/f4702fb2e5c54e189b3430b90cd93ef8 as hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/f0252adfc27b54a76a05585f3f45bf90/A/f4702fb2e5c54e189b3430b90cd93ef8 2024-11-24T22:14:03,104 INFO [RS_FLUSH_OPERATIONS-regionserver/bd6592c37bb2:0-2 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/f0252adfc27b54a76a05585f3f45bf90/A/f4702fb2e5c54e189b3430b90cd93ef8, entries=150, sequenceid=501, filesize=12.0 K 2024-11-24T22:14:03,105 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bd6592c37bb2:0-2 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/f0252adfc27b54a76a05585f3f45bf90/.tmp/B/7ddf65b17c1948cb8dac4dce2ff012c6 as hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/f0252adfc27b54a76a05585f3f45bf90/B/7ddf65b17c1948cb8dac4dce2ff012c6 2024-11-24T22:14:03,112 INFO [RS_FLUSH_OPERATIONS-regionserver/bd6592c37bb2:0-2 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/f0252adfc27b54a76a05585f3f45bf90/B/7ddf65b17c1948cb8dac4dce2ff012c6, entries=150, sequenceid=501, filesize=12.0 K 2024-11-24T22:14:03,113 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bd6592c37bb2:0-2 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/f0252adfc27b54a76a05585f3f45bf90/.tmp/C/fc8735a74d5548058f328073367c7532 as hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/f0252adfc27b54a76a05585f3f45bf90/C/fc8735a74d5548058f328073367c7532 2024-11-24T22:14:03,120 INFO [RS_FLUSH_OPERATIONS-regionserver/bd6592c37bb2:0-2 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/f0252adfc27b54a76a05585f3f45bf90/C/fc8735a74d5548058f328073367c7532, entries=150, sequenceid=501, filesize=12.0 K 2024-11-24T22:14:03,121 INFO [RS_FLUSH_OPERATIONS-regionserver/bd6592c37bb2:0-2 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.HRegion(3140): Finished flush of dataSize ~100.63 KB/103050, heapSize ~299.53 KB/306720, currentSize=0 B/0 for f0252adfc27b54a76a05585f3f45bf90 in 531ms, sequenceid=501, compaction requested=false 2024-11-24T22:14:03,121 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bd6592c37bb2:0-2 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.HRegion(2603): Flush status journal for f0252adfc27b54a76a05585f3f45bf90: 2024-11-24T22:14:03,121 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bd6592c37bb2:0-2 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732486426439.f0252adfc27b54a76a05585f3f45bf90. 2024-11-24T22:14:03,121 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bd6592c37bb2:0-2 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=16 2024-11-24T22:14:03,122 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37701 {}] master.HMaster(4169): Remote procedure done, pid=16 2024-11-24T22:14:03,129 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=16, resume processing ppid=15 2024-11-24T22:14:03,129 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=16, ppid=15, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.7810 sec 2024-11-24T22:14:03,135 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=15, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=15, table=TestAcidGuarantees in 1.7890 sec 2024-11-24T22:14:03,314 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40087 {}] regionserver.HRegion(8855): Flush requested on f0252adfc27b54a76a05585f3f45bf90 2024-11-24T22:14:03,314 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing f0252adfc27b54a76a05585f3f45bf90 3/3 column families, dataSize=46.96 KB heapSize=140.20 KB 2024-11-24T22:14:03,314 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK f0252adfc27b54a76a05585f3f45bf90, store=A 2024-11-24T22:14:03,315 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-24T22:14:03,315 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK f0252adfc27b54a76a05585f3f45bf90, store=B 2024-11-24T22:14:03,315 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-24T22:14:03,315 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK f0252adfc27b54a76a05585f3f45bf90, store=C 2024-11-24T22:14:03,315 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-24T22:14:03,327 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/f0252adfc27b54a76a05585f3f45bf90/.tmp/A/c6b8d474fb464a6dbbb00cd9d0d18449 is 50, key is test_row_0/A:col10/1732486443305/Put/seqid=0 2024-11-24T22:14:03,333 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45929 is added to blk_1073741894_1070 (size=12301) 2024-11-24T22:14:03,333 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=512 (bloomFilter=true), to=hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/f0252adfc27b54a76a05585f3f45bf90/.tmp/A/c6b8d474fb464a6dbbb00cd9d0d18449 2024-11-24T22:14:03,353 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/f0252adfc27b54a76a05585f3f45bf90/.tmp/B/70abb3621bf148439ad48345232947f0 is 50, key is test_row_0/B:col10/1732486443305/Put/seqid=0 2024-11-24T22:14:03,360 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45929 is added to blk_1073741895_1071 (size=12301) 2024-11-24T22:14:03,362 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=512 (bloomFilter=true), to=hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/f0252adfc27b54a76a05585f3f45bf90/.tmp/B/70abb3621bf148439ad48345232947f0 2024-11-24T22:14:03,375 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/f0252adfc27b54a76a05585f3f45bf90/.tmp/C/7c396145845b4013bbcfdadc15e47807 is 50, key is test_row_0/C:col10/1732486443305/Put/seqid=0 2024-11-24T22:14:03,397 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/bd6592c37bb2:0-3 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=259, cellSize=11855, totalHeapSize=27176, min timestamp=1732486443363, max timestamp=1732486443393Num uniques 859; 2024-11-24T22:14:03,397 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/bd6592c37bb2:0-3 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=11855, suffix heap size=117064,new segment heap size=27176 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=259 2024-11-24T22:14:03,397 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/bd6592c37bb2:0-1 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=259, cellSize=11855, totalHeapSize=27176, min timestamp=1732486443363, max timestamp=1732486443393Num uniques 859; 2024-11-24T22:14:03,397 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/bd6592c37bb2:0-0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=259, cellSize=11855, totalHeapSize=27176, min timestamp=1732486443363, max timestamp=1732486443393Num uniques 859; 2024-11-24T22:14:03,397 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/bd6592c37bb2:0-1 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=11855, suffix heap size=117064,new segment heap size=27176 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=259 2024-11-24T22:14:03,397 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/bd6592c37bb2:0-0 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=11855, suffix heap size=117064,new segment heap size=27176 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=259 2024-11-24T22:14:03,398 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45929 is added to blk_1073741896_1072 (size=12301) 2024-11-24T22:14:03,463 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40087 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0252adfc27b54a76a05585f3f45bf90, server=bd6592c37bb2,40087,1732486423713 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-24T22:14:03,464 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40087 {}] ipc.CallRunner(138): callId: 189 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36518 deadline: 1732486503462, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0252adfc27b54a76a05585f3f45bf90, server=bd6592c37bb2,40087,1732486423713 2024-11-24T22:14:03,464 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732486426439.f0252adfc27b54a76a05585f3f45bf90., hostname=bd6592c37bb2,40087,1732486423713, seqNum=2 , the old value is region=TestAcidGuarantees,,1732486426439.f0252adfc27b54a76a05585f3f45bf90., hostname=bd6592c37bb2,40087,1732486423713, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0252adfc27b54a76a05585f3f45bf90, server=bd6592c37bb2,40087,1732486423713 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T22:14:03,464 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732486426439.f0252adfc27b54a76a05585f3f45bf90., hostname=bd6592c37bb2,40087,1732486423713, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0252adfc27b54a76a05585f3f45bf90, server=bd6592c37bb2,40087,1732486423713 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T22:14:03,464 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732486426439.f0252adfc27b54a76a05585f3f45bf90., hostname=bd6592c37bb2,40087,1732486423713, seqNum=2 because the exception is null or not the one we care about 2024-11-24T22:14:03,472 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40087 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0252adfc27b54a76a05585f3f45bf90, server=bd6592c37bb2,40087,1732486423713 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-24T22:14:03,473 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40087 {}] ipc.CallRunner(138): callId: 167 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36532 deadline: 1732486503464, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0252adfc27b54a76a05585f3f45bf90, server=bd6592c37bb2,40087,1732486423713 2024-11-24T22:14:03,473 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732486426439.f0252adfc27b54a76a05585f3f45bf90., hostname=bd6592c37bb2,40087,1732486423713, seqNum=2 , the old value is region=TestAcidGuarantees,,1732486426439.f0252adfc27b54a76a05585f3f45bf90., hostname=bd6592c37bb2,40087,1732486423713, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0252adfc27b54a76a05585f3f45bf90, server=bd6592c37bb2,40087,1732486423713 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T22:14:03,473 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732486426439.f0252adfc27b54a76a05585f3f45bf90., hostname=bd6592c37bb2,40087,1732486423713, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0252adfc27b54a76a05585f3f45bf90, server=bd6592c37bb2,40087,1732486423713 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T22:14:03,473 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732486426439.f0252adfc27b54a76a05585f3f45bf90., hostname=bd6592c37bb2,40087,1732486423713, seqNum=2 because the exception is null or not the one we care about 2024-11-24T22:14:03,490 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37701 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=15 2024-11-24T22:14:03,491 INFO [RPCClient-NioEventLoopGroup-4-8 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-24T22:14:03,491 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37701 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-24T22:14:03,492 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37701 {}] procedure2.ProcedureExecutor(1139): Stored pid=17, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=17, table=TestAcidGuarantees 2024-11-24T22:14:03,493 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37701 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=17 2024-11-24T22:14:03,493 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=17, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=17, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-24T22:14:03,494 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=17, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=17, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-24T22:14:03,494 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=18, ppid=17, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-24T22:14:03,572 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40087 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0252adfc27b54a76a05585f3f45bf90, server=bd6592c37bb2,40087,1732486423713 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-24T22:14:03,572 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40087 {}] ipc.CallRunner(138): callId: 190 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36518 deadline: 1732486503571, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0252adfc27b54a76a05585f3f45bf90, server=bd6592c37bb2,40087,1732486423713 2024-11-24T22:14:03,572 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732486426439.f0252adfc27b54a76a05585f3f45bf90., hostname=bd6592c37bb2,40087,1732486423713, seqNum=2 , the old value is region=TestAcidGuarantees,,1732486426439.f0252adfc27b54a76a05585f3f45bf90., hostname=bd6592c37bb2,40087,1732486423713, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0252adfc27b54a76a05585f3f45bf90, server=bd6592c37bb2,40087,1732486423713 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T22:14:03,572 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732486426439.f0252adfc27b54a76a05585f3f45bf90., hostname=bd6592c37bb2,40087,1732486423713, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0252adfc27b54a76a05585f3f45bf90, server=bd6592c37bb2,40087,1732486423713 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T22:14:03,572 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732486426439.f0252adfc27b54a76a05585f3f45bf90., hostname=bd6592c37bb2,40087,1732486423713, seqNum=2 because the exception is null or not the one we care about 2024-11-24T22:14:03,584 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40087 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0252adfc27b54a76a05585f3f45bf90, server=bd6592c37bb2,40087,1732486423713 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-24T22:14:03,585 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40087 {}] ipc.CallRunner(138): callId: 168 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36532 deadline: 1732486503581, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0252adfc27b54a76a05585f3f45bf90, server=bd6592c37bb2,40087,1732486423713 2024-11-24T22:14:03,585 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732486426439.f0252adfc27b54a76a05585f3f45bf90., hostname=bd6592c37bb2,40087,1732486423713, seqNum=2 , the old value is region=TestAcidGuarantees,,1732486426439.f0252adfc27b54a76a05585f3f45bf90., hostname=bd6592c37bb2,40087,1732486423713, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0252adfc27b54a76a05585f3f45bf90, server=bd6592c37bb2,40087,1732486423713 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T22:14:03,585 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732486426439.f0252adfc27b54a76a05585f3f45bf90., hostname=bd6592c37bb2,40087,1732486423713, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0252adfc27b54a76a05585f3f45bf90, server=bd6592c37bb2,40087,1732486423713 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T22:14:03,585 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732486426439.f0252adfc27b54a76a05585f3f45bf90., hostname=bd6592c37bb2,40087,1732486423713, seqNum=2 because the exception is null or not the one we care about 2024-11-24T22:14:03,601 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37701 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=17 2024-11-24T22:14:03,647 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=40087 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=18 2024-11-24T22:14:03,648 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bd6592c37bb2:0-0 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732486426439.f0252adfc27b54a76a05585f3f45bf90. 2024-11-24T22:14:03,648 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bd6592c37bb2:0-0 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732486426439.f0252adfc27b54a76a05585f3f45bf90. as already flushing 2024-11-24T22:14:03,648 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bd6592c37bb2:0-0 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732486426439.f0252adfc27b54a76a05585f3f45bf90. 2024-11-24T22:14:03,648 ERROR [RS_FLUSH_OPERATIONS-regionserver/bd6592c37bb2:0-0 {event_type=RS_FLUSH_REGIONS, pid=18}] handler.RSProcedureHandler(58): pid=18 java.io.IOException: Unable to complete flush {ENCODED => f0252adfc27b54a76a05585f3f45bf90, NAME => 'TestAcidGuarantees,,1732486426439.f0252adfc27b54a76a05585f3f45bf90.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T22:14:03,648 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bd6592c37bb2:0-0 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=18 java.io.IOException: Unable to complete flush {ENCODED => f0252adfc27b54a76a05585f3f45bf90, NAME => 'TestAcidGuarantees,,1732486426439.f0252adfc27b54a76a05585f3f45bf90.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T22:14:03,649 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37701 {}] master.HMaster(4177): Remote procedure failed, pid=18 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => f0252adfc27b54a76a05585f3f45bf90, NAME => 'TestAcidGuarantees,,1732486426439.f0252adfc27b54a76a05585f3f45bf90.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => f0252adfc27b54a76a05585f3f45bf90, NAME => 'TestAcidGuarantees,,1732486426439.f0252adfc27b54a76a05585f3f45bf90.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T22:14:03,789 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40087 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0252adfc27b54a76a05585f3f45bf90, server=bd6592c37bb2,40087,1732486423713 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-24T22:14:03,789 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40087 {}] ipc.CallRunner(138): callId: 191 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36518 deadline: 1732486503780, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0252adfc27b54a76a05585f3f45bf90, server=bd6592c37bb2,40087,1732486423713 2024-11-24T22:14:03,790 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732486426439.f0252adfc27b54a76a05585f3f45bf90., hostname=bd6592c37bb2,40087,1732486423713, seqNum=2 , the old value is region=TestAcidGuarantees,,1732486426439.f0252adfc27b54a76a05585f3f45bf90., hostname=bd6592c37bb2,40087,1732486423713, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0252adfc27b54a76a05585f3f45bf90, server=bd6592c37bb2,40087,1732486423713 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T22:14:03,790 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732486426439.f0252adfc27b54a76a05585f3f45bf90., hostname=bd6592c37bb2,40087,1732486423713, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0252adfc27b54a76a05585f3f45bf90, server=bd6592c37bb2,40087,1732486423713 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T22:14:03,790 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732486426439.f0252adfc27b54a76a05585f3f45bf90., hostname=bd6592c37bb2,40087,1732486423713, seqNum=2 because the exception is null or not the one we care about 2024-11-24T22:14:03,797 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=512 (bloomFilter=true), to=hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/f0252adfc27b54a76a05585f3f45bf90/.tmp/C/7c396145845b4013bbcfdadc15e47807 2024-11-24T22:14:03,801 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=40087 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=18 2024-11-24T22:14:03,793 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40087 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0252adfc27b54a76a05585f3f45bf90, server=bd6592c37bb2,40087,1732486423713 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-24T22:14:03,801 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bd6592c37bb2:0-1 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732486426439.f0252adfc27b54a76a05585f3f45bf90. 2024-11-24T22:14:03,801 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40087 {}] ipc.CallRunner(138): callId: 169 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36532 deadline: 1732486503791, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0252adfc27b54a76a05585f3f45bf90, server=bd6592c37bb2,40087,1732486423713 2024-11-24T22:14:03,801 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bd6592c37bb2:0-1 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732486426439.f0252adfc27b54a76a05585f3f45bf90. as already flushing 2024-11-24T22:14:03,801 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bd6592c37bb2:0-1 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732486426439.f0252adfc27b54a76a05585f3f45bf90. 2024-11-24T22:14:03,801 ERROR [RS_FLUSH_OPERATIONS-regionserver/bd6592c37bb2:0-1 {event_type=RS_FLUSH_REGIONS, pid=18}] handler.RSProcedureHandler(58): pid=18 java.io.IOException: Unable to complete flush {ENCODED => f0252adfc27b54a76a05585f3f45bf90, NAME => 'TestAcidGuarantees,,1732486426439.f0252adfc27b54a76a05585f3f45bf90.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T22:14:03,801 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bd6592c37bb2:0-1 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=18 java.io.IOException: Unable to complete flush {ENCODED => f0252adfc27b54a76a05585f3f45bf90, NAME => 'TestAcidGuarantees,,1732486426439.f0252adfc27b54a76a05585f3f45bf90.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T22:14:03,804 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37701 {}] master.HMaster(4177): Remote procedure failed, pid=18 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => f0252adfc27b54a76a05585f3f45bf90, NAME => 'TestAcidGuarantees,,1732486426439.f0252adfc27b54a76a05585f3f45bf90.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => f0252adfc27b54a76a05585f3f45bf90, NAME => 'TestAcidGuarantees,,1732486426439.f0252adfc27b54a76a05585f3f45bf90.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T22:14:03,805 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732486426439.f0252adfc27b54a76a05585f3f45bf90., hostname=bd6592c37bb2,40087,1732486423713, seqNum=2 , the old value is region=TestAcidGuarantees,,1732486426439.f0252adfc27b54a76a05585f3f45bf90., hostname=bd6592c37bb2,40087,1732486423713, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0252adfc27b54a76a05585f3f45bf90, server=bd6592c37bb2,40087,1732486423713 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T22:14:03,805 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732486426439.f0252adfc27b54a76a05585f3f45bf90., hostname=bd6592c37bb2,40087,1732486423713, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0252adfc27b54a76a05585f3f45bf90, server=bd6592c37bb2,40087,1732486423713 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T22:14:03,805 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732486426439.f0252adfc27b54a76a05585f3f45bf90., hostname=bd6592c37bb2,40087,1732486423713, seqNum=2 because the exception is null or not the one we care about 2024-11-24T22:14:03,809 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/f0252adfc27b54a76a05585f3f45bf90/.tmp/A/c6b8d474fb464a6dbbb00cd9d0d18449 as hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/f0252adfc27b54a76a05585f3f45bf90/A/c6b8d474fb464a6dbbb00cd9d0d18449 2024-11-24T22:14:03,811 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37701 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=17 2024-11-24T22:14:03,822 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/f0252adfc27b54a76a05585f3f45bf90/A/c6b8d474fb464a6dbbb00cd9d0d18449, entries=150, sequenceid=512, filesize=12.0 K 2024-11-24T22:14:03,824 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/f0252adfc27b54a76a05585f3f45bf90/.tmp/B/70abb3621bf148439ad48345232947f0 as hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/f0252adfc27b54a76a05585f3f45bf90/B/70abb3621bf148439ad48345232947f0 2024-11-24T22:14:03,831 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/f0252adfc27b54a76a05585f3f45bf90/B/70abb3621bf148439ad48345232947f0, entries=150, sequenceid=512, filesize=12.0 K 2024-11-24T22:14:03,832 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/f0252adfc27b54a76a05585f3f45bf90/.tmp/C/7c396145845b4013bbcfdadc15e47807 as hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/f0252adfc27b54a76a05585f3f45bf90/C/7c396145845b4013bbcfdadc15e47807 2024-11-24T22:14:03,838 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/f0252adfc27b54a76a05585f3f45bf90/C/7c396145845b4013bbcfdadc15e47807, entries=150, sequenceid=512, filesize=12.0 K 2024-11-24T22:14:03,839 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~46.96 KB/48090, heapSize ~140.16 KB/143520, currentSize=134.18 KB/137400 for f0252adfc27b54a76a05585f3f45bf90 in 525ms, sequenceid=512, compaction requested=true 2024-11-24T22:14:03,839 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for f0252adfc27b54a76a05585f3f45bf90: 2024-11-24T22:14:03,839 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store f0252adfc27b54a76a05585f3f45bf90:A, priority=-2147483648, current under compaction store size is 1 2024-11-24T22:14:03,839 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-24T22:14:03,839 DEBUG [RS:0;bd6592c37bb2:40087-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-24T22:14:03,839 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store f0252adfc27b54a76a05585f3f45bf90:B, priority=-2147483648, current under compaction store size is 2 2024-11-24T22:14:03,839 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-24T22:14:03,840 DEBUG [RS:0;bd6592c37bb2:40087-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-24T22:14:03,840 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store f0252adfc27b54a76a05585f3f45bf90:C, priority=-2147483648, current under compaction store size is 3 2024-11-24T22:14:03,840 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-24T22:14:03,841 DEBUG [RS:0;bd6592c37bb2:40087-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37483 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-24T22:14:03,841 DEBUG [RS:0;bd6592c37bb2:40087-longCompactions-0 {}] regionserver.HStore(1541): f0252adfc27b54a76a05585f3f45bf90/B is initiating minor compaction (all files) 2024-11-24T22:14:03,841 DEBUG [RS:0;bd6592c37bb2:40087-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37483 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-24T22:14:03,841 DEBUG [RS:0;bd6592c37bb2:40087-shortCompactions-0 {}] regionserver.HStore(1541): f0252adfc27b54a76a05585f3f45bf90/A is initiating minor compaction (all files) 2024-11-24T22:14:03,841 INFO [RS:0;bd6592c37bb2:40087-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of f0252adfc27b54a76a05585f3f45bf90/B in TestAcidGuarantees,,1732486426439.f0252adfc27b54a76a05585f3f45bf90. 2024-11-24T22:14:03,841 INFO [RS:0;bd6592c37bb2:40087-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of f0252adfc27b54a76a05585f3f45bf90/A in TestAcidGuarantees,,1732486426439.f0252adfc27b54a76a05585f3f45bf90. 2024-11-24T22:14:03,841 INFO [RS:0;bd6592c37bb2:40087-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/f0252adfc27b54a76a05585f3f45bf90/B/f786dc1420844a9da33ca12b1b422ea3, hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/f0252adfc27b54a76a05585f3f45bf90/B/7ddf65b17c1948cb8dac4dce2ff012c6, hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/f0252adfc27b54a76a05585f3f45bf90/B/70abb3621bf148439ad48345232947f0] into tmpdir=hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/f0252adfc27b54a76a05585f3f45bf90/.tmp, totalSize=36.6 K 2024-11-24T22:14:03,841 INFO [RS:0;bd6592c37bb2:40087-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/f0252adfc27b54a76a05585f3f45bf90/A/1dc58facd3964ecdb9a0f4fffee4cfe6, hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/f0252adfc27b54a76a05585f3f45bf90/A/f4702fb2e5c54e189b3430b90cd93ef8, hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/f0252adfc27b54a76a05585f3f45bf90/A/c6b8d474fb464a6dbbb00cd9d0d18449] into tmpdir=hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/f0252adfc27b54a76a05585f3f45bf90/.tmp, totalSize=36.6 K 2024-11-24T22:14:03,842 DEBUG [RS:0;bd6592c37bb2:40087-longCompactions-0 {}] compactions.Compactor(225): Compacting f786dc1420844a9da33ca12b1b422ea3, keycount=150, bloomtype=ROW, size=12.6 K, encoding=NONE, compression=NONE, seqNum=481, earliestPutTs=1732486439980 2024-11-24T22:14:03,842 DEBUG [RS:0;bd6592c37bb2:40087-shortCompactions-0 {}] compactions.Compactor(225): Compacting 1dc58facd3964ecdb9a0f4fffee4cfe6, keycount=150, bloomtype=ROW, size=12.6 K, encoding=NONE, compression=NONE, seqNum=481, earliestPutTs=1732486439980 2024-11-24T22:14:03,842 DEBUG [RS:0;bd6592c37bb2:40087-longCompactions-0 {}] compactions.Compactor(225): Compacting 7ddf65b17c1948cb8dac4dce2ff012c6, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=501, earliestPutTs=1732486441139 2024-11-24T22:14:03,842 DEBUG [RS:0;bd6592c37bb2:40087-shortCompactions-0 {}] compactions.Compactor(225): Compacting f4702fb2e5c54e189b3430b90cd93ef8, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=501, earliestPutTs=1732486441139 2024-11-24T22:14:03,843 DEBUG [RS:0;bd6592c37bb2:40087-longCompactions-0 {}] compactions.Compactor(225): Compacting 70abb3621bf148439ad48345232947f0, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=512, earliestPutTs=1732486443305 2024-11-24T22:14:03,843 DEBUG [RS:0;bd6592c37bb2:40087-shortCompactions-0 {}] compactions.Compactor(225): Compacting c6b8d474fb464a6dbbb00cd9d0d18449, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=512, earliestPutTs=1732486443305 2024-11-24T22:14:03,852 INFO [RS:0;bd6592c37bb2:40087-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): f0252adfc27b54a76a05585f3f45bf90#A#compaction#60 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-24T22:14:03,853 DEBUG [RS:0;bd6592c37bb2:40087-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/f0252adfc27b54a76a05585f3f45bf90/.tmp/A/7e8bd1915fc94a819d890f3dcfe52f79 is 50, key is test_row_0/A:col10/1732486443305/Put/seqid=0 2024-11-24T22:14:03,854 INFO [RS:0;bd6592c37bb2:40087-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): f0252adfc27b54a76a05585f3f45bf90#B#compaction#61 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-24T22:14:03,855 DEBUG [RS:0;bd6592c37bb2:40087-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/f0252adfc27b54a76a05585f3f45bf90/.tmp/B/da93b0c6087f4675a21ee70eb6f87149 is 50, key is test_row_0/B:col10/1732486443305/Put/seqid=0 2024-11-24T22:14:03,862 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45929 is added to blk_1073741897_1073 (size=12983) 2024-11-24T22:14:03,864 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45929 is added to blk_1073741898_1074 (size=12983) 2024-11-24T22:14:03,871 DEBUG [RS:0;bd6592c37bb2:40087-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/f0252adfc27b54a76a05585f3f45bf90/.tmp/A/7e8bd1915fc94a819d890f3dcfe52f79 as hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/f0252adfc27b54a76a05585f3f45bf90/A/7e8bd1915fc94a819d890f3dcfe52f79 2024-11-24T22:14:03,879 INFO [RS:0;bd6592c37bb2:40087-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in f0252adfc27b54a76a05585f3f45bf90/A of f0252adfc27b54a76a05585f3f45bf90 into 7e8bd1915fc94a819d890f3dcfe52f79(size=12.7 K), total size for store is 12.7 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-24T22:14:03,879 DEBUG [RS:0;bd6592c37bb2:40087-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for f0252adfc27b54a76a05585f3f45bf90: 2024-11-24T22:14:03,879 INFO [RS:0;bd6592c37bb2:40087-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732486426439.f0252adfc27b54a76a05585f3f45bf90., storeName=f0252adfc27b54a76a05585f3f45bf90/A, priority=13, startTime=1732486443839; duration=0sec 2024-11-24T22:14:03,880 DEBUG [RS:0;bd6592c37bb2:40087-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-24T22:14:03,880 DEBUG [RS:0;bd6592c37bb2:40087-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: f0252adfc27b54a76a05585f3f45bf90:A 2024-11-24T22:14:03,880 DEBUG [RS:0;bd6592c37bb2:40087-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-24T22:14:03,881 DEBUG [RS:0;bd6592c37bb2:40087-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37483 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-24T22:14:03,881 DEBUG [RS:0;bd6592c37bb2:40087-shortCompactions-0 {}] regionserver.HStore(1541): f0252adfc27b54a76a05585f3f45bf90/C is initiating minor compaction (all files) 2024-11-24T22:14:03,881 INFO [RS:0;bd6592c37bb2:40087-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of f0252adfc27b54a76a05585f3f45bf90/C in TestAcidGuarantees,,1732486426439.f0252adfc27b54a76a05585f3f45bf90. 2024-11-24T22:14:03,881 INFO [RS:0;bd6592c37bb2:40087-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/f0252adfc27b54a76a05585f3f45bf90/C/984a805e6d2a48d3b398a98cf43f97d2, hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/f0252adfc27b54a76a05585f3f45bf90/C/fc8735a74d5548058f328073367c7532, hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/f0252adfc27b54a76a05585f3f45bf90/C/7c396145845b4013bbcfdadc15e47807] into tmpdir=hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/f0252adfc27b54a76a05585f3f45bf90/.tmp, totalSize=36.6 K 2024-11-24T22:14:03,882 DEBUG [RS:0;bd6592c37bb2:40087-shortCompactions-0 {}] compactions.Compactor(225): Compacting 984a805e6d2a48d3b398a98cf43f97d2, keycount=150, bloomtype=ROW, size=12.6 K, encoding=NONE, compression=NONE, seqNum=481, earliestPutTs=1732486439980 2024-11-24T22:14:03,882 DEBUG [RS:0;bd6592c37bb2:40087-shortCompactions-0 {}] compactions.Compactor(225): Compacting fc8735a74d5548058f328073367c7532, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=501, earliestPutTs=1732486441139 2024-11-24T22:14:03,886 DEBUG [RS:0;bd6592c37bb2:40087-shortCompactions-0 {}] compactions.Compactor(225): Compacting 7c396145845b4013bbcfdadc15e47807, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=512, earliestPutTs=1732486443305 2024-11-24T22:14:03,897 INFO [RS:0;bd6592c37bb2:40087-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): f0252adfc27b54a76a05585f3f45bf90#C#compaction#62 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-24T22:14:03,898 DEBUG [RS:0;bd6592c37bb2:40087-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/f0252adfc27b54a76a05585f3f45bf90/.tmp/C/05ada7a4ea8043d293e29bd7672e3405 is 50, key is test_row_0/C:col10/1732486443305/Put/seqid=0 2024-11-24T22:14:03,905 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45929 is added to blk_1073741899_1075 (size=12983) 2024-11-24T22:14:03,957 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=40087 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=18 2024-11-24T22:14:03,958 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bd6592c37bb2:0-2 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732486426439.f0252adfc27b54a76a05585f3f45bf90. 2024-11-24T22:14:03,958 INFO [RS_FLUSH_OPERATIONS-regionserver/bd6592c37bb2:0-2 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.HRegion(2902): Flushing f0252adfc27b54a76a05585f3f45bf90 3/3 column families, dataSize=134.18 KB heapSize=375.61 KB 2024-11-24T22:14:03,958 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bd6592c37bb2:0-2 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.CompactingMemStore(205): FLUSHING TO DISK f0252adfc27b54a76a05585f3f45bf90, store=A 2024-11-24T22:14:03,958 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bd6592c37bb2:0-2 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-24T22:14:03,958 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bd6592c37bb2:0-2 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.CompactingMemStore(205): FLUSHING TO DISK f0252adfc27b54a76a05585f3f45bf90, store=B 2024-11-24T22:14:03,958 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bd6592c37bb2:0-2 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-24T22:14:03,958 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bd6592c37bb2:0-2 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.CompactingMemStore(205): FLUSHING TO DISK f0252adfc27b54a76a05585f3f45bf90, store=C 2024-11-24T22:14:03,958 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bd6592c37bb2:0-2 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-24T22:14:03,965 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bd6592c37bb2:0-2 {event_type=RS_FLUSH_REGIONS, pid=18}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/f0252adfc27b54a76a05585f3f45bf90/.tmp/A/749bc91e4eac4c2894066ef9bc5b64b6 is 50, key is test_row_0/A:col10/1732486443461/Put/seqid=0 2024-11-24T22:14:03,974 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45929 is added to blk_1073741900_1076 (size=12301) 2024-11-24T22:14:04,105 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40087 {}] regionserver.HRegion(8855): Flush requested on f0252adfc27b54a76a05585f3f45bf90 2024-11-24T22:14:04,105 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732486426439.f0252adfc27b54a76a05585f3f45bf90. as already flushing 2024-11-24T22:14:04,121 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37701 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=17 2024-11-24T22:14:04,125 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40087 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0252adfc27b54a76a05585f3f45bf90, server=bd6592c37bb2,40087,1732486423713 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-24T22:14:04,125 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40087 {}] ipc.CallRunner(138): callId: 172 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36532 deadline: 1732486504124, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0252adfc27b54a76a05585f3f45bf90, server=bd6592c37bb2,40087,1732486423713 2024-11-24T22:14:04,126 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732486426439.f0252adfc27b54a76a05585f3f45bf90., hostname=bd6592c37bb2,40087,1732486423713, seqNum=2 , the old value is region=TestAcidGuarantees,,1732486426439.f0252adfc27b54a76a05585f3f45bf90., hostname=bd6592c37bb2,40087,1732486423713, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0252adfc27b54a76a05585f3f45bf90, server=bd6592c37bb2,40087,1732486423713 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T22:14:04,126 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732486426439.f0252adfc27b54a76a05585f3f45bf90., hostname=bd6592c37bb2,40087,1732486423713, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0252adfc27b54a76a05585f3f45bf90, server=bd6592c37bb2,40087,1732486423713 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T22:14:04,126 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732486426439.f0252adfc27b54a76a05585f3f45bf90., hostname=bd6592c37bb2,40087,1732486423713, seqNum=2 because the exception is null or not the one we care about 2024-11-24T22:14:04,128 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40087 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0252adfc27b54a76a05585f3f45bf90, server=bd6592c37bb2,40087,1732486423713 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-24T22:14:04,128 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40087 {}] ipc.CallRunner(138): callId: 197 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36518 deadline: 1732486504125, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0252adfc27b54a76a05585f3f45bf90, server=bd6592c37bb2,40087,1732486423713 2024-11-24T22:14:04,129 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732486426439.f0252adfc27b54a76a05585f3f45bf90., hostname=bd6592c37bb2,40087,1732486423713, seqNum=2 , the old value is region=TestAcidGuarantees,,1732486426439.f0252adfc27b54a76a05585f3f45bf90., hostname=bd6592c37bb2,40087,1732486423713, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0252adfc27b54a76a05585f3f45bf90, server=bd6592c37bb2,40087,1732486423713 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T22:14:04,129 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732486426439.f0252adfc27b54a76a05585f3f45bf90., hostname=bd6592c37bb2,40087,1732486423713, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0252adfc27b54a76a05585f3f45bf90, server=bd6592c37bb2,40087,1732486423713 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T22:14:04,129 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732486426439.f0252adfc27b54a76a05585f3f45bf90., hostname=bd6592c37bb2,40087,1732486423713, seqNum=2 because the exception is null or not the one we care about 2024-11-24T22:14:04,193 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40087 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0252adfc27b54a76a05585f3f45bf90, server=bd6592c37bb2,40087,1732486423713 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-24T22:14:04,194 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40087 {}] ipc.CallRunner(138): callId: 107 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36590 deadline: 1732486504191, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0252adfc27b54a76a05585f3f45bf90, server=bd6592c37bb2,40087,1732486423713 2024-11-24T22:14:04,194 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40087 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0252adfc27b54a76a05585f3f45bf90, server=bd6592c37bb2,40087,1732486423713 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-24T22:14:04,194 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40087 {}] ipc.CallRunner(138): callId: 120 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36562 deadline: 1732486504191, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0252adfc27b54a76a05585f3f45bf90, server=bd6592c37bb2,40087,1732486423713 2024-11-24T22:14:04,194 WARN [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRpcRetryingCaller(177): Call to bd6592c37bb2,40087,1732486423713 for 'test_row_1' in f0252adfc27b54a76a05585f3f45bf90 of TestAcidGuarantees failed, tries = 7, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 4195 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0252adfc27b54a76a05585f3f45bf90, server=bd6592c37bb2,40087,1732486423713 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor44.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0252adfc27b54a76a05585f3f45bf90, server=bd6592c37bb2,40087,1732486423713 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-24T22:14:04,195 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732486426439.f0252adfc27b54a76a05585f3f45bf90., hostname=bd6592c37bb2,40087,1732486423713, seqNum=2 , the old value is region=TestAcidGuarantees,,1732486426439.f0252adfc27b54a76a05585f3f45bf90., hostname=bd6592c37bb2,40087,1732486423713, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0252adfc27b54a76a05585f3f45bf90, server=bd6592c37bb2,40087,1732486423713 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T22:14:04,195 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732486426439.f0252adfc27b54a76a05585f3f45bf90., hostname=bd6592c37bb2,40087,1732486423713, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0252adfc27b54a76a05585f3f45bf90, server=bd6592c37bb2,40087,1732486423713 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T22:14:04,195 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732486426439.f0252adfc27b54a76a05585f3f45bf90., hostname=bd6592c37bb2,40087,1732486423713, seqNum=2 because the exception is null or not the one we care about 2024-11-24T22:14:04,194 WARN [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRpcRetryingCaller(177): Call to bd6592c37bb2,40087,1732486423713 for 'test_row_0' in f0252adfc27b54a76a05585f3f45bf90 of TestAcidGuarantees failed, tries = 7, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 4190 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0252adfc27b54a76a05585f3f45bf90, server=bd6592c37bb2,40087,1732486423713 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor44.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0252adfc27b54a76a05585f3f45bf90, server=bd6592c37bb2,40087,1732486423713 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-24T22:14:04,195 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732486426439.f0252adfc27b54a76a05585f3f45bf90., hostname=bd6592c37bb2,40087,1732486423713, seqNum=2 , the old value is region=TestAcidGuarantees,,1732486426439.f0252adfc27b54a76a05585f3f45bf90., hostname=bd6592c37bb2,40087,1732486423713, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0252adfc27b54a76a05585f3f45bf90, server=bd6592c37bb2,40087,1732486423713 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T22:14:04,195 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732486426439.f0252adfc27b54a76a05585f3f45bf90., hostname=bd6592c37bb2,40087,1732486423713, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0252adfc27b54a76a05585f3f45bf90, server=bd6592c37bb2,40087,1732486423713 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T22:14:04,195 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732486426439.f0252adfc27b54a76a05585f3f45bf90., hostname=bd6592c37bb2,40087,1732486423713, seqNum=2 because the exception is null or not the one we care about 2024-11-24T22:14:04,201 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40087 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0252adfc27b54a76a05585f3f45bf90, server=bd6592c37bb2,40087,1732486423713 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-24T22:14:04,202 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40087 {}] ipc.CallRunner(138): callId: 101 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36548 deadline: 1732486504201, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0252adfc27b54a76a05585f3f45bf90, server=bd6592c37bb2,40087,1732486423713 2024-11-24T22:14:04,202 WARN [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRpcRetryingCaller(177): Call to bd6592c37bb2,40087,1732486423713 for 'test_row_0' in f0252adfc27b54a76a05585f3f45bf90 of TestAcidGuarantees failed, tries = 7, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 4198 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0252adfc27b54a76a05585f3f45bf90, server=bd6592c37bb2,40087,1732486423713 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor44.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0252adfc27b54a76a05585f3f45bf90, server=bd6592c37bb2,40087,1732486423713 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-24T22:14:04,203 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732486426439.f0252adfc27b54a76a05585f3f45bf90., hostname=bd6592c37bb2,40087,1732486423713, seqNum=2 , the old value is region=TestAcidGuarantees,,1732486426439.f0252adfc27b54a76a05585f3f45bf90., hostname=bd6592c37bb2,40087,1732486423713, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0252adfc27b54a76a05585f3f45bf90, server=bd6592c37bb2,40087,1732486423713 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T22:14:04,203 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732486426439.f0252adfc27b54a76a05585f3f45bf90., hostname=bd6592c37bb2,40087,1732486423713, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0252adfc27b54a76a05585f3f45bf90, server=bd6592c37bb2,40087,1732486423713 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T22:14:04,203 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732486426439.f0252adfc27b54a76a05585f3f45bf90., hostname=bd6592c37bb2,40087,1732486423713, seqNum=2 because the exception is null or not the one we care about 2024-11-24T22:14:04,232 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40087 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0252adfc27b54a76a05585f3f45bf90, server=bd6592c37bb2,40087,1732486423713 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-24T22:14:04,232 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40087 {}] ipc.CallRunner(138): callId: 173 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36532 deadline: 1732486504230, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0252adfc27b54a76a05585f3f45bf90, server=bd6592c37bb2,40087,1732486423713 2024-11-24T22:14:04,233 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40087 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0252adfc27b54a76a05585f3f45bf90, server=bd6592c37bb2,40087,1732486423713 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-24T22:14:04,233 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40087 {}] ipc.CallRunner(138): callId: 198 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36518 deadline: 1732486504231, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0252adfc27b54a76a05585f3f45bf90, server=bd6592c37bb2,40087,1732486423713 2024-11-24T22:14:04,233 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732486426439.f0252adfc27b54a76a05585f3f45bf90., hostname=bd6592c37bb2,40087,1732486423713, seqNum=2 , the old value is region=TestAcidGuarantees,,1732486426439.f0252adfc27b54a76a05585f3f45bf90., hostname=bd6592c37bb2,40087,1732486423713, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0252adfc27b54a76a05585f3f45bf90, server=bd6592c37bb2,40087,1732486423713 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T22:14:04,233 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732486426439.f0252adfc27b54a76a05585f3f45bf90., hostname=bd6592c37bb2,40087,1732486423713, seqNum=2 , the old value is region=TestAcidGuarantees,,1732486426439.f0252adfc27b54a76a05585f3f45bf90., hostname=bd6592c37bb2,40087,1732486423713, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0252adfc27b54a76a05585f3f45bf90, server=bd6592c37bb2,40087,1732486423713 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T22:14:04,233 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732486426439.f0252adfc27b54a76a05585f3f45bf90., hostname=bd6592c37bb2,40087,1732486423713, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0252adfc27b54a76a05585f3f45bf90, server=bd6592c37bb2,40087,1732486423713 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T22:14:04,233 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732486426439.f0252adfc27b54a76a05585f3f45bf90., hostname=bd6592c37bb2,40087,1732486423713, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0252adfc27b54a76a05585f3f45bf90, server=bd6592c37bb2,40087,1732486423713 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T22:14:04,233 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732486426439.f0252adfc27b54a76a05585f3f45bf90., hostname=bd6592c37bb2,40087,1732486423713, seqNum=2 because the exception is null or not the one we care about 2024-11-24T22:14:04,233 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732486426439.f0252adfc27b54a76a05585f3f45bf90., hostname=bd6592c37bb2,40087,1732486423713, seqNum=2 because the exception is null or not the one we care about 2024-11-24T22:14:04,272 DEBUG [RS:0;bd6592c37bb2:40087-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/f0252adfc27b54a76a05585f3f45bf90/.tmp/B/da93b0c6087f4675a21ee70eb6f87149 as hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/f0252adfc27b54a76a05585f3f45bf90/B/da93b0c6087f4675a21ee70eb6f87149 2024-11-24T22:14:04,280 INFO [RS:0;bd6592c37bb2:40087-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in f0252adfc27b54a76a05585f3f45bf90/B of f0252adfc27b54a76a05585f3f45bf90 into da93b0c6087f4675a21ee70eb6f87149(size=12.7 K), total size for store is 12.7 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-24T22:14:04,280 DEBUG [RS:0;bd6592c37bb2:40087-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for f0252adfc27b54a76a05585f3f45bf90: 2024-11-24T22:14:04,280 INFO [RS:0;bd6592c37bb2:40087-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732486426439.f0252adfc27b54a76a05585f3f45bf90., storeName=f0252adfc27b54a76a05585f3f45bf90/B, priority=13, startTime=1732486443839; duration=0sec 2024-11-24T22:14:04,280 DEBUG [RS:0;bd6592c37bb2:40087-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-24T22:14:04,281 DEBUG [RS:0;bd6592c37bb2:40087-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: f0252adfc27b54a76a05585f3f45bf90:B 2024-11-24T22:14:04,312 DEBUG [RS:0;bd6592c37bb2:40087-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/f0252adfc27b54a76a05585f3f45bf90/.tmp/C/05ada7a4ea8043d293e29bd7672e3405 as hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/f0252adfc27b54a76a05585f3f45bf90/C/05ada7a4ea8043d293e29bd7672e3405 2024-11-24T22:14:04,319 INFO [RS:0;bd6592c37bb2:40087-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in f0252adfc27b54a76a05585f3f45bf90/C of f0252adfc27b54a76a05585f3f45bf90 into 05ada7a4ea8043d293e29bd7672e3405(size=12.7 K), total size for store is 12.7 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-24T22:14:04,319 DEBUG [RS:0;bd6592c37bb2:40087-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for f0252adfc27b54a76a05585f3f45bf90: 2024-11-24T22:14:04,319 INFO [RS:0;bd6592c37bb2:40087-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732486426439.f0252adfc27b54a76a05585f3f45bf90., storeName=f0252adfc27b54a76a05585f3f45bf90/C, priority=13, startTime=1732486443840; duration=0sec 2024-11-24T22:14:04,319 DEBUG [RS:0;bd6592c37bb2:40087-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-24T22:14:04,319 DEBUG [RS:0;bd6592c37bb2:40087-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: f0252adfc27b54a76a05585f3f45bf90:C 2024-11-24T22:14:04,375 INFO [RS_FLUSH_OPERATIONS-regionserver/bd6592c37bb2:0-2 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=548 (bloomFilter=true), to=hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/f0252adfc27b54a76a05585f3f45bf90/.tmp/A/749bc91e4eac4c2894066ef9bc5b64b6 2024-11-24T22:14:04,407 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bd6592c37bb2:0-2 {event_type=RS_FLUSH_REGIONS, pid=18}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/f0252adfc27b54a76a05585f3f45bf90/.tmp/B/e63699926e974340b82e6c44beff995b is 50, key is test_row_0/B:col10/1732486443461/Put/seqid=0 2024-11-24T22:14:04,421 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45929 is added to blk_1073741901_1077 (size=12301) 2024-11-24T22:14:04,443 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40087 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0252adfc27b54a76a05585f3f45bf90, server=bd6592c37bb2,40087,1732486423713 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-24T22:14:04,444 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40087 {}] ipc.CallRunner(138): callId: 199 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36518 deadline: 1732486504441, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0252adfc27b54a76a05585f3f45bf90, server=bd6592c37bb2,40087,1732486423713 2024-11-24T22:14:04,444 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40087 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0252adfc27b54a76a05585f3f45bf90, server=bd6592c37bb2,40087,1732486423713 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-24T22:14:04,445 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40087 {}] ipc.CallRunner(138): callId: 174 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36532 deadline: 1732486504441, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0252adfc27b54a76a05585f3f45bf90, server=bd6592c37bb2,40087,1732486423713 2024-11-24T22:14:04,445 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732486426439.f0252adfc27b54a76a05585f3f45bf90., hostname=bd6592c37bb2,40087,1732486423713, seqNum=2 , the old value is region=TestAcidGuarantees,,1732486426439.f0252adfc27b54a76a05585f3f45bf90., hostname=bd6592c37bb2,40087,1732486423713, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0252adfc27b54a76a05585f3f45bf90, server=bd6592c37bb2,40087,1732486423713 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T22:14:04,445 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732486426439.f0252adfc27b54a76a05585f3f45bf90., hostname=bd6592c37bb2,40087,1732486423713, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0252adfc27b54a76a05585f3f45bf90, server=bd6592c37bb2,40087,1732486423713 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T22:14:04,445 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732486426439.f0252adfc27b54a76a05585f3f45bf90., hostname=bd6592c37bb2,40087,1732486423713, seqNum=2 because the exception is null or not the one we care about 2024-11-24T22:14:04,445 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732486426439.f0252adfc27b54a76a05585f3f45bf90., hostname=bd6592c37bb2,40087,1732486423713, seqNum=2 , the old value is region=TestAcidGuarantees,,1732486426439.f0252adfc27b54a76a05585f3f45bf90., hostname=bd6592c37bb2,40087,1732486423713, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0252adfc27b54a76a05585f3f45bf90, server=bd6592c37bb2,40087,1732486423713 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T22:14:04,445 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732486426439.f0252adfc27b54a76a05585f3f45bf90., hostname=bd6592c37bb2,40087,1732486423713, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0252adfc27b54a76a05585f3f45bf90, server=bd6592c37bb2,40087,1732486423713 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T22:14:04,445 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732486426439.f0252adfc27b54a76a05585f3f45bf90., hostname=bd6592c37bb2,40087,1732486423713, seqNum=2 because the exception is null or not the one we care about 2024-11-24T22:14:04,647 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37701 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=17 2024-11-24T22:14:04,753 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40087 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0252adfc27b54a76a05585f3f45bf90, server=bd6592c37bb2,40087,1732486423713 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-24T22:14:04,754 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40087 {}] ipc.CallRunner(138): callId: 200 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36518 deadline: 1732486504751, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0252adfc27b54a76a05585f3f45bf90, server=bd6592c37bb2,40087,1732486423713 2024-11-24T22:14:04,754 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40087 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0252adfc27b54a76a05585f3f45bf90, server=bd6592c37bb2,40087,1732486423713 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-24T22:14:04,754 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732486426439.f0252adfc27b54a76a05585f3f45bf90., hostname=bd6592c37bb2,40087,1732486423713, seqNum=2 , the old value is region=TestAcidGuarantees,,1732486426439.f0252adfc27b54a76a05585f3f45bf90., hostname=bd6592c37bb2,40087,1732486423713, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0252adfc27b54a76a05585f3f45bf90, server=bd6592c37bb2,40087,1732486423713 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T22:14:04,754 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732486426439.f0252adfc27b54a76a05585f3f45bf90., hostname=bd6592c37bb2,40087,1732486423713, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0252adfc27b54a76a05585f3f45bf90, server=bd6592c37bb2,40087,1732486423713 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T22:14:04,754 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40087 {}] ipc.CallRunner(138): callId: 175 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36532 deadline: 1732486504751, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0252adfc27b54a76a05585f3f45bf90, server=bd6592c37bb2,40087,1732486423713 2024-11-24T22:14:04,754 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732486426439.f0252adfc27b54a76a05585f3f45bf90., hostname=bd6592c37bb2,40087,1732486423713, seqNum=2 because the exception is null or not the one we care about 2024-11-24T22:14:04,755 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732486426439.f0252adfc27b54a76a05585f3f45bf90., hostname=bd6592c37bb2,40087,1732486423713, seqNum=2 , the old value is region=TestAcidGuarantees,,1732486426439.f0252adfc27b54a76a05585f3f45bf90., hostname=bd6592c37bb2,40087,1732486423713, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0252adfc27b54a76a05585f3f45bf90, server=bd6592c37bb2,40087,1732486423713 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T22:14:04,755 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732486426439.f0252adfc27b54a76a05585f3f45bf90., hostname=bd6592c37bb2,40087,1732486423713, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0252adfc27b54a76a05585f3f45bf90, server=bd6592c37bb2,40087,1732486423713 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T22:14:04,755 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732486426439.f0252adfc27b54a76a05585f3f45bf90., hostname=bd6592c37bb2,40087,1732486423713, seqNum=2 because the exception is null or not the one we care about 2024-11-24T22:14:04,823 INFO [RS_FLUSH_OPERATIONS-regionserver/bd6592c37bb2:0-2 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=548 (bloomFilter=true), to=hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/f0252adfc27b54a76a05585f3f45bf90/.tmp/B/e63699926e974340b82e6c44beff995b 2024-11-24T22:14:04,847 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bd6592c37bb2:0-2 {event_type=RS_FLUSH_REGIONS, pid=18}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/f0252adfc27b54a76a05585f3f45bf90/.tmp/C/96d7872a2b26416798bfbc2e3cf792d4 is 50, key is test_row_0/C:col10/1732486443461/Put/seqid=0 2024-11-24T22:14:04,910 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45929 is added to blk_1073741902_1078 (size=12301) 2024-11-24T22:14:04,912 INFO [RS_FLUSH_OPERATIONS-regionserver/bd6592c37bb2:0-2 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=548 (bloomFilter=true), to=hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/f0252adfc27b54a76a05585f3f45bf90/.tmp/C/96d7872a2b26416798bfbc2e3cf792d4 2024-11-24T22:14:04,925 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bd6592c37bb2:0-2 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/f0252adfc27b54a76a05585f3f45bf90/.tmp/A/749bc91e4eac4c2894066ef9bc5b64b6 as hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/f0252adfc27b54a76a05585f3f45bf90/A/749bc91e4eac4c2894066ef9bc5b64b6 2024-11-24T22:14:04,935 INFO [RS_FLUSH_OPERATIONS-regionserver/bd6592c37bb2:0-2 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/f0252adfc27b54a76a05585f3f45bf90/A/749bc91e4eac4c2894066ef9bc5b64b6, entries=150, sequenceid=548, filesize=12.0 K 2024-11-24T22:14:04,937 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bd6592c37bb2:0-2 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/f0252adfc27b54a76a05585f3f45bf90/.tmp/B/e63699926e974340b82e6c44beff995b as hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/f0252adfc27b54a76a05585f3f45bf90/B/e63699926e974340b82e6c44beff995b 2024-11-24T22:14:04,949 INFO [RS_FLUSH_OPERATIONS-regionserver/bd6592c37bb2:0-2 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/f0252adfc27b54a76a05585f3f45bf90/B/e63699926e974340b82e6c44beff995b, entries=150, sequenceid=548, filesize=12.0 K 2024-11-24T22:14:04,951 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bd6592c37bb2:0-2 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/f0252adfc27b54a76a05585f3f45bf90/.tmp/C/96d7872a2b26416798bfbc2e3cf792d4 as hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/f0252adfc27b54a76a05585f3f45bf90/C/96d7872a2b26416798bfbc2e3cf792d4 2024-11-24T22:14:04,958 INFO [RS_FLUSH_OPERATIONS-regionserver/bd6592c37bb2:0-2 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/f0252adfc27b54a76a05585f3f45bf90/C/96d7872a2b26416798bfbc2e3cf792d4, entries=150, sequenceid=548, filesize=12.0 K 2024-11-24T22:14:04,960 INFO [RS_FLUSH_OPERATIONS-regionserver/bd6592c37bb2:0-2 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.HRegion(3140): Finished flush of dataSize ~134.18 KB/137400, heapSize ~375.56 KB/384576, currentSize=46.96 KB/48090 for f0252adfc27b54a76a05585f3f45bf90 in 1002ms, sequenceid=548, compaction requested=false 2024-11-24T22:14:04,960 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bd6592c37bb2:0-2 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.HRegion(2603): Flush status journal for f0252adfc27b54a76a05585f3f45bf90: 2024-11-24T22:14:04,960 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bd6592c37bb2:0-2 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732486426439.f0252adfc27b54a76a05585f3f45bf90. 2024-11-24T22:14:04,960 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bd6592c37bb2:0-2 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=18 2024-11-24T22:14:04,961 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37701 {}] master.HMaster(4169): Remote procedure done, pid=18 2024-11-24T22:14:04,964 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=18, resume processing ppid=17 2024-11-24T22:14:04,964 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=18, ppid=17, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.4680 sec 2024-11-24T22:14:04,972 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=17, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=17, table=TestAcidGuarantees in 1.4780 sec 2024-11-24T22:14:05,268 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40087 {}] regionserver.HRegion(8855): Flush requested on f0252adfc27b54a76a05585f3f45bf90 2024-11-24T22:14:05,268 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing f0252adfc27b54a76a05585f3f45bf90 3/3 column families, dataSize=53.67 KB heapSize=160.13 KB 2024-11-24T22:14:05,271 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK f0252adfc27b54a76a05585f3f45bf90, store=A 2024-11-24T22:14:05,272 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-24T22:14:05,272 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK f0252adfc27b54a76a05585f3f45bf90, store=B 2024-11-24T22:14:05,272 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-24T22:14:05,272 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK f0252adfc27b54a76a05585f3f45bf90, store=C 2024-11-24T22:14:05,272 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-24T22:14:05,283 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/f0252adfc27b54a76a05585f3f45bf90/.tmp/A/1a617bcb02d94acf9f84238fe3511506 is 50, key is test_row_0/A:col10/1732486445267/Put/seqid=0 2024-11-24T22:14:05,330 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45929 is added to blk_1073741903_1079 (size=17181) 2024-11-24T22:14:05,332 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=562 (bloomFilter=true), to=hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/f0252adfc27b54a76a05585f3f45bf90/.tmp/A/1a617bcb02d94acf9f84238fe3511506 2024-11-24T22:14:05,354 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/f0252adfc27b54a76a05585f3f45bf90/.tmp/B/4d4d753c5b6141dbba6ad07b969acd4b is 50, key is test_row_0/B:col10/1732486445267/Put/seqid=0 2024-11-24T22:14:05,361 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40087 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0252adfc27b54a76a05585f3f45bf90, server=bd6592c37bb2,40087,1732486423713 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-24T22:14:05,361 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40087 {}] ipc.CallRunner(138): callId: 211 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36518 deadline: 1732486505354, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0252adfc27b54a76a05585f3f45bf90, server=bd6592c37bb2,40087,1732486423713 2024-11-24T22:14:05,362 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40087 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0252adfc27b54a76a05585f3f45bf90, server=bd6592c37bb2,40087,1732486423713 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-24T22:14:05,362 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732486426439.f0252adfc27b54a76a05585f3f45bf90., hostname=bd6592c37bb2,40087,1732486423713, seqNum=2 , the old value is region=TestAcidGuarantees,,1732486426439.f0252adfc27b54a76a05585f3f45bf90., hostname=bd6592c37bb2,40087,1732486423713, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0252adfc27b54a76a05585f3f45bf90, server=bd6592c37bb2,40087,1732486423713 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T22:14:05,362 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40087 {}] ipc.CallRunner(138): callId: 185 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36532 deadline: 1732486505355, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0252adfc27b54a76a05585f3f45bf90, server=bd6592c37bb2,40087,1732486423713 2024-11-24T22:14:05,362 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732486426439.f0252adfc27b54a76a05585f3f45bf90., hostname=bd6592c37bb2,40087,1732486423713, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0252adfc27b54a76a05585f3f45bf90, server=bd6592c37bb2,40087,1732486423713 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T22:14:05,362 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732486426439.f0252adfc27b54a76a05585f3f45bf90., hostname=bd6592c37bb2,40087,1732486423713, seqNum=2 because the exception is null or not the one we care about 2024-11-24T22:14:05,362 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732486426439.f0252adfc27b54a76a05585f3f45bf90., hostname=bd6592c37bb2,40087,1732486423713, seqNum=2 , the old value is region=TestAcidGuarantees,,1732486426439.f0252adfc27b54a76a05585f3f45bf90., hostname=bd6592c37bb2,40087,1732486423713, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0252adfc27b54a76a05585f3f45bf90, server=bd6592c37bb2,40087,1732486423713 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T22:14:05,362 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732486426439.f0252adfc27b54a76a05585f3f45bf90., hostname=bd6592c37bb2,40087,1732486423713, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0252adfc27b54a76a05585f3f45bf90, server=bd6592c37bb2,40087,1732486423713 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T22:14:05,362 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732486426439.f0252adfc27b54a76a05585f3f45bf90., hostname=bd6592c37bb2,40087,1732486423713, seqNum=2 because the exception is null or not the one we care about 2024-11-24T22:14:05,393 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45929 is added to blk_1073741904_1080 (size=12301) 2024-11-24T22:14:05,394 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=562 (bloomFilter=true), to=hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/f0252adfc27b54a76a05585f3f45bf90/.tmp/B/4d4d753c5b6141dbba6ad07b969acd4b 2024-11-24T22:14:05,418 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/f0252adfc27b54a76a05585f3f45bf90/.tmp/C/a92a99c7f4e04ab09097ec22773f3fd9 is 50, key is test_row_0/C:col10/1732486445267/Put/seqid=0 2024-11-24T22:14:05,431 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45929 is added to blk_1073741905_1081 (size=12301) 2024-11-24T22:14:05,432 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=562 (bloomFilter=true), to=hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/f0252adfc27b54a76a05585f3f45bf90/.tmp/C/a92a99c7f4e04ab09097ec22773f3fd9 2024-11-24T22:14:05,440 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/f0252adfc27b54a76a05585f3f45bf90/.tmp/A/1a617bcb02d94acf9f84238fe3511506 as hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/f0252adfc27b54a76a05585f3f45bf90/A/1a617bcb02d94acf9f84238fe3511506 2024-11-24T22:14:05,450 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/f0252adfc27b54a76a05585f3f45bf90/A/1a617bcb02d94acf9f84238fe3511506, entries=250, sequenceid=562, filesize=16.8 K 2024-11-24T22:14:05,452 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/f0252adfc27b54a76a05585f3f45bf90/.tmp/B/4d4d753c5b6141dbba6ad07b969acd4b as hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/f0252adfc27b54a76a05585f3f45bf90/B/4d4d753c5b6141dbba6ad07b969acd4b 2024-11-24T22:14:05,461 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/f0252adfc27b54a76a05585f3f45bf90/B/4d4d753c5b6141dbba6ad07b969acd4b, entries=150, sequenceid=562, filesize=12.0 K 2024-11-24T22:14:05,463 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/f0252adfc27b54a76a05585f3f45bf90/.tmp/C/a92a99c7f4e04ab09097ec22773f3fd9 as hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/f0252adfc27b54a76a05585f3f45bf90/C/a92a99c7f4e04ab09097ec22773f3fd9 2024-11-24T22:14:05,472 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40087 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0252adfc27b54a76a05585f3f45bf90, server=bd6592c37bb2,40087,1732486423713 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-24T22:14:05,473 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40087 {}] ipc.CallRunner(138): callId: 212 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36518 deadline: 1732486505471, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0252adfc27b54a76a05585f3f45bf90, server=bd6592c37bb2,40087,1732486423713 2024-11-24T22:14:05,473 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732486426439.f0252adfc27b54a76a05585f3f45bf90., hostname=bd6592c37bb2,40087,1732486423713, seqNum=2 , the old value is region=TestAcidGuarantees,,1732486426439.f0252adfc27b54a76a05585f3f45bf90., hostname=bd6592c37bb2,40087,1732486423713, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0252adfc27b54a76a05585f3f45bf90, server=bd6592c37bb2,40087,1732486423713 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T22:14:05,473 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732486426439.f0252adfc27b54a76a05585f3f45bf90., hostname=bd6592c37bb2,40087,1732486423713, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0252adfc27b54a76a05585f3f45bf90, server=bd6592c37bb2,40087,1732486423713 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T22:14:05,473 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732486426439.f0252adfc27b54a76a05585f3f45bf90., hostname=bd6592c37bb2,40087,1732486423713, seqNum=2 because the exception is null or not the one we care about 2024-11-24T22:14:05,474 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40087 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0252adfc27b54a76a05585f3f45bf90, server=bd6592c37bb2,40087,1732486423713 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-24T22:14:05,475 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40087 {}] ipc.CallRunner(138): callId: 186 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36532 deadline: 1732486505471, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0252adfc27b54a76a05585f3f45bf90, server=bd6592c37bb2,40087,1732486423713 2024-11-24T22:14:05,475 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732486426439.f0252adfc27b54a76a05585f3f45bf90., hostname=bd6592c37bb2,40087,1732486423713, seqNum=2 , the old value is region=TestAcidGuarantees,,1732486426439.f0252adfc27b54a76a05585f3f45bf90., hostname=bd6592c37bb2,40087,1732486423713, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0252adfc27b54a76a05585f3f45bf90, server=bd6592c37bb2,40087,1732486423713 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T22:14:05,475 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732486426439.f0252adfc27b54a76a05585f3f45bf90., hostname=bd6592c37bb2,40087,1732486423713, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0252adfc27b54a76a05585f3f45bf90, server=bd6592c37bb2,40087,1732486423713 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T22:14:05,475 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732486426439.f0252adfc27b54a76a05585f3f45bf90., hostname=bd6592c37bb2,40087,1732486423713, seqNum=2 because the exception is null or not the one we care about 2024-11-24T22:14:05,477 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/f0252adfc27b54a76a05585f3f45bf90/C/a92a99c7f4e04ab09097ec22773f3fd9, entries=150, sequenceid=562, filesize=12.0 K 2024-11-24T22:14:05,478 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~60.38 KB/61830, heapSize ~180 KB/184320, currentSize=114.05 KB/116790 for f0252adfc27b54a76a05585f3f45bf90 in 210ms, sequenceid=562, compaction requested=true 2024-11-24T22:14:05,478 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for f0252adfc27b54a76a05585f3f45bf90: 2024-11-24T22:14:05,479 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store f0252adfc27b54a76a05585f3f45bf90:A, priority=-2147483648, current under compaction store size is 1 2024-11-24T22:14:05,479 DEBUG [RS:0;bd6592c37bb2:40087-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-24T22:14:05,479 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-24T22:14:05,479 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store f0252adfc27b54a76a05585f3f45bf90:B, priority=-2147483648, current under compaction store size is 2 2024-11-24T22:14:05,479 DEBUG [RS:0;bd6592c37bb2:40087-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-24T22:14:05,479 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-24T22:14:05,479 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store f0252adfc27b54a76a05585f3f45bf90:C, priority=-2147483648, current under compaction store size is 3 2024-11-24T22:14:05,479 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-24T22:14:05,481 DEBUG [RS:0;bd6592c37bb2:40087-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 42465 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-24T22:14:05,481 DEBUG [RS:0;bd6592c37bb2:40087-shortCompactions-0 {}] regionserver.HStore(1541): f0252adfc27b54a76a05585f3f45bf90/A is initiating minor compaction (all files) 2024-11-24T22:14:05,481 INFO [RS:0;bd6592c37bb2:40087-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of f0252adfc27b54a76a05585f3f45bf90/A in TestAcidGuarantees,,1732486426439.f0252adfc27b54a76a05585f3f45bf90. 2024-11-24T22:14:05,481 INFO [RS:0;bd6592c37bb2:40087-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/f0252adfc27b54a76a05585f3f45bf90/A/7e8bd1915fc94a819d890f3dcfe52f79, hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/f0252adfc27b54a76a05585f3f45bf90/A/749bc91e4eac4c2894066ef9bc5b64b6, hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/f0252adfc27b54a76a05585f3f45bf90/A/1a617bcb02d94acf9f84238fe3511506] into tmpdir=hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/f0252adfc27b54a76a05585f3f45bf90/.tmp, totalSize=41.5 K 2024-11-24T22:14:05,482 DEBUG [RS:0;bd6592c37bb2:40087-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37585 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-24T22:14:05,482 DEBUG [RS:0;bd6592c37bb2:40087-longCompactions-0 {}] regionserver.HStore(1541): f0252adfc27b54a76a05585f3f45bf90/B is initiating minor compaction (all files) 2024-11-24T22:14:05,482 INFO [RS:0;bd6592c37bb2:40087-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of f0252adfc27b54a76a05585f3f45bf90/B in TestAcidGuarantees,,1732486426439.f0252adfc27b54a76a05585f3f45bf90. 2024-11-24T22:14:05,482 INFO [RS:0;bd6592c37bb2:40087-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/f0252adfc27b54a76a05585f3f45bf90/B/da93b0c6087f4675a21ee70eb6f87149, hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/f0252adfc27b54a76a05585f3f45bf90/B/e63699926e974340b82e6c44beff995b, hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/f0252adfc27b54a76a05585f3f45bf90/B/4d4d753c5b6141dbba6ad07b969acd4b] into tmpdir=hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/f0252adfc27b54a76a05585f3f45bf90/.tmp, totalSize=36.7 K 2024-11-24T22:14:05,482 DEBUG [RS:0;bd6592c37bb2:40087-shortCompactions-0 {}] compactions.Compactor(225): Compacting 7e8bd1915fc94a819d890f3dcfe52f79, keycount=150, bloomtype=ROW, size=12.7 K, encoding=NONE, compression=NONE, seqNum=512, earliestPutTs=1732486443305 2024-11-24T22:14:05,483 DEBUG [RS:0;bd6592c37bb2:40087-longCompactions-0 {}] compactions.Compactor(225): Compacting da93b0c6087f4675a21ee70eb6f87149, keycount=150, bloomtype=ROW, size=12.7 K, encoding=NONE, compression=NONE, seqNum=512, earliestPutTs=1732486443305 2024-11-24T22:14:05,483 DEBUG [RS:0;bd6592c37bb2:40087-shortCompactions-0 {}] compactions.Compactor(225): Compacting 749bc91e4eac4c2894066ef9bc5b64b6, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=548, earliestPutTs=1732486443438 2024-11-24T22:14:05,484 DEBUG [RS:0;bd6592c37bb2:40087-longCompactions-0 {}] compactions.Compactor(225): Compacting e63699926e974340b82e6c44beff995b, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=548, earliestPutTs=1732486443438 2024-11-24T22:14:05,484 DEBUG [RS:0;bd6592c37bb2:40087-shortCompactions-0 {}] compactions.Compactor(225): Compacting 1a617bcb02d94acf9f84238fe3511506, keycount=250, bloomtype=ROW, size=16.8 K, encoding=NONE, compression=NONE, seqNum=562, earliestPutTs=1732486444119 2024-11-24T22:14:05,485 DEBUG [RS:0;bd6592c37bb2:40087-longCompactions-0 {}] compactions.Compactor(225): Compacting 4d4d753c5b6141dbba6ad07b969acd4b, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=562, earliestPutTs=1732486444123 2024-11-24T22:14:05,504 INFO [RS:0;bd6592c37bb2:40087-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): f0252adfc27b54a76a05585f3f45bf90#A#compaction#69 average throughput is 2.18 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-24T22:14:05,505 DEBUG [RS:0;bd6592c37bb2:40087-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/f0252adfc27b54a76a05585f3f45bf90/.tmp/A/da0dc8e8af434e93b77478eb2c1ab439 is 50, key is test_row_0/A:col10/1732486445267/Put/seqid=0 2024-11-24T22:14:05,505 INFO [RS:0;bd6592c37bb2:40087-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): f0252adfc27b54a76a05585f3f45bf90#B#compaction#70 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-24T22:14:05,506 DEBUG [RS:0;bd6592c37bb2:40087-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/f0252adfc27b54a76a05585f3f45bf90/.tmp/B/cc7c51615aeb4494be1ea60782c5134a is 50, key is test_row_0/B:col10/1732486445267/Put/seqid=0 2024-11-24T22:14:05,520 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45929 is added to blk_1073741906_1082 (size=13085) 2024-11-24T22:14:05,521 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45929 is added to blk_1073741907_1083 (size=13085) 2024-11-24T22:14:05,651 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37701 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=17 2024-11-24T22:14:05,652 INFO [RPCClient-NioEventLoopGroup-4-8 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-24T22:14:05,652 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37701 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-24T22:14:05,653 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37701 {}] procedure2.ProcedureExecutor(1139): Stored pid=19, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=19, table=TestAcidGuarantees 2024-11-24T22:14:05,654 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37701 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=19 2024-11-24T22:14:05,655 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=19, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=19, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-24T22:14:05,659 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=19, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=19, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-24T22:14:05,659 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=20, ppid=19, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-24T22:14:05,687 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/bd6592c37bb2:0-7 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1732486445346, max timestamp=1732486445681Num uniques 859; 2024-11-24T22:14:05,687 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/bd6592c37bb2:0-7 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-11-24T22:14:05,687 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/bd6592c37bb2:0-4 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1732486445346, max timestamp=1732486445683Num uniques 859; 2024-11-24T22:14:05,687 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/bd6592c37bb2:0-4 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-11-24T22:14:05,690 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/bd6592c37bb2:0-8 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1732486445346, max timestamp=1732486445681Num uniques 859; 2024-11-24T22:14:05,690 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/bd6592c37bb2:0-8 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-11-24T22:14:05,702 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40087 {}] regionserver.HRegion(8855): Flush requested on f0252adfc27b54a76a05585f3f45bf90 2024-11-24T22:14:05,702 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing f0252adfc27b54a76a05585f3f45bf90 3/3 column families, dataSize=53.67 KB heapSize=145.92 KB 2024-11-24T22:14:05,702 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK f0252adfc27b54a76a05585f3f45bf90, store=A 2024-11-24T22:14:05,702 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-24T22:14:05,703 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK f0252adfc27b54a76a05585f3f45bf90, store=B 2024-11-24T22:14:05,703 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-24T22:14:05,703 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK f0252adfc27b54a76a05585f3f45bf90, store=C 2024-11-24T22:14:05,703 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-24T22:14:05,708 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/f0252adfc27b54a76a05585f3f45bf90/.tmp/A/bb4373777b7749fb9ffb570fede7c5e7 is 50, key is test_row_0/A:col10/1732486445698/Put/seqid=0 2024-11-24T22:14:05,713 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45929 is added to blk_1073741908_1084 (size=17181) 2024-11-24T22:14:05,714 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=588 (bloomFilter=true), to=hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/f0252adfc27b54a76a05585f3f45bf90/.tmp/A/bb4373777b7749fb9ffb570fede7c5e7 2024-11-24T22:14:05,726 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/f0252adfc27b54a76a05585f3f45bf90/.tmp/B/13d49dd09c36477fa2ccd8d4b6050657 is 50, key is test_row_0/B:col10/1732486445698/Put/seqid=0 2024-11-24T22:14:05,761 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37701 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=19 2024-11-24T22:14:05,770 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45929 is added to blk_1073741909_1085 (size=12301) 2024-11-24T22:14:05,771 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=588 (bloomFilter=true), to=hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/f0252adfc27b54a76a05585f3f45bf90/.tmp/B/13d49dd09c36477fa2ccd8d4b6050657 2024-11-24T22:14:05,782 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/f0252adfc27b54a76a05585f3f45bf90/.tmp/C/3e8950a3d04e473f9a791f9f71875f4e is 50, key is test_row_0/C:col10/1732486445698/Put/seqid=0 2024-11-24T22:14:05,806 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/bd6592c37bb2:0-6 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=259, cellSize=11855, totalHeapSize=27176, min timestamp=1732486445767, max timestamp=1732486445802Num uniques 859; 2024-11-24T22:14:05,806 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/bd6592c37bb2:0-6 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=11855, suffix heap size=117064,new segment heap size=27176 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=259 2024-11-24T22:14:05,810 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/bd6592c37bb2:0-9 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1732486445783, max timestamp=1732486445802Num uniques 859; 2024-11-24T22:14:05,810 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/bd6592c37bb2:0-9 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-11-24T22:14:05,813 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/bd6592c37bb2:0-2 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1732486445783, max timestamp=1732486445802Num uniques 859; 2024-11-24T22:14:05,813 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/bd6592c37bb2:0-2 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-11-24T22:14:05,820 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=40087 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=20 2024-11-24T22:14:05,820 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bd6592c37bb2:0-0 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732486426439.f0252adfc27b54a76a05585f3f45bf90. 2024-11-24T22:14:05,820 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bd6592c37bb2:0-0 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732486426439.f0252adfc27b54a76a05585f3f45bf90. as already flushing 2024-11-24T22:14:05,820 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bd6592c37bb2:0-0 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732486426439.f0252adfc27b54a76a05585f3f45bf90. 2024-11-24T22:14:05,821 ERROR [RS_FLUSH_OPERATIONS-regionserver/bd6592c37bb2:0-0 {event_type=RS_FLUSH_REGIONS, pid=20}] handler.RSProcedureHandler(58): pid=20 java.io.IOException: Unable to complete flush {ENCODED => f0252adfc27b54a76a05585f3f45bf90, NAME => 'TestAcidGuarantees,,1732486426439.f0252adfc27b54a76a05585f3f45bf90.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T22:14:05,821 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bd6592c37bb2:0-0 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=20 java.io.IOException: Unable to complete flush {ENCODED => f0252adfc27b54a76a05585f3f45bf90, NAME => 'TestAcidGuarantees,,1732486426439.f0252adfc27b54a76a05585f3f45bf90.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T22:14:05,823 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45929 is added to blk_1073741910_1086 (size=12301) 2024-11-24T22:14:05,824 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37701 {}] master.HMaster(4177): Remote procedure failed, pid=20 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => f0252adfc27b54a76a05585f3f45bf90, NAME => 'TestAcidGuarantees,,1732486426439.f0252adfc27b54a76a05585f3f45bf90.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => f0252adfc27b54a76a05585f3f45bf90, NAME => 'TestAcidGuarantees,,1732486426439.f0252adfc27b54a76a05585f3f45bf90.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T22:14:05,898 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40087 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0252adfc27b54a76a05585f3f45bf90, server=bd6592c37bb2,40087,1732486423713 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-24T22:14:05,898 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40087 {}] ipc.CallRunner(138): callId: 232 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36518 deadline: 1732486505881, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0252adfc27b54a76a05585f3f45bf90, server=bd6592c37bb2,40087,1732486423713 2024-11-24T22:14:05,901 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732486426439.f0252adfc27b54a76a05585f3f45bf90., hostname=bd6592c37bb2,40087,1732486423713, seqNum=2 , the old value is region=TestAcidGuarantees,,1732486426439.f0252adfc27b54a76a05585f3f45bf90., hostname=bd6592c37bb2,40087,1732486423713, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0252adfc27b54a76a05585f3f45bf90, server=bd6592c37bb2,40087,1732486423713 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T22:14:05,901 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732486426439.f0252adfc27b54a76a05585f3f45bf90., hostname=bd6592c37bb2,40087,1732486423713, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0252adfc27b54a76a05585f3f45bf90, server=bd6592c37bb2,40087,1732486423713 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T22:14:05,901 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732486426439.f0252adfc27b54a76a05585f3f45bf90., hostname=bd6592c37bb2,40087,1732486423713, seqNum=2 because the exception is null or not the one we care about 2024-11-24T22:14:05,904 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40087 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0252adfc27b54a76a05585f3f45bf90, server=bd6592c37bb2,40087,1732486423713 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-24T22:14:05,904 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40087 {}] ipc.CallRunner(138): callId: 206 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36532 deadline: 1732486505897, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0252adfc27b54a76a05585f3f45bf90, server=bd6592c37bb2,40087,1732486423713 2024-11-24T22:14:05,905 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732486426439.f0252adfc27b54a76a05585f3f45bf90., hostname=bd6592c37bb2,40087,1732486423713, seqNum=2 , the old value is region=TestAcidGuarantees,,1732486426439.f0252adfc27b54a76a05585f3f45bf90., hostname=bd6592c37bb2,40087,1732486423713, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0252adfc27b54a76a05585f3f45bf90, server=bd6592c37bb2,40087,1732486423713 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T22:14:05,905 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732486426439.f0252adfc27b54a76a05585f3f45bf90., hostname=bd6592c37bb2,40087,1732486423713, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0252adfc27b54a76a05585f3f45bf90, server=bd6592c37bb2,40087,1732486423713 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T22:14:05,905 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732486426439.f0252adfc27b54a76a05585f3f45bf90., hostname=bd6592c37bb2,40087,1732486423713, seqNum=2 because the exception is null or not the one we care about 2024-11-24T22:14:05,932 DEBUG [RS:0;bd6592c37bb2:40087-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/f0252adfc27b54a76a05585f3f45bf90/.tmp/A/da0dc8e8af434e93b77478eb2c1ab439 as hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/f0252adfc27b54a76a05585f3f45bf90/A/da0dc8e8af434e93b77478eb2c1ab439 2024-11-24T22:14:05,935 DEBUG [RS:0;bd6592c37bb2:40087-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/f0252adfc27b54a76a05585f3f45bf90/.tmp/B/cc7c51615aeb4494be1ea60782c5134a as hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/f0252adfc27b54a76a05585f3f45bf90/B/cc7c51615aeb4494be1ea60782c5134a 2024-11-24T22:14:05,945 INFO [RS:0;bd6592c37bb2:40087-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in f0252adfc27b54a76a05585f3f45bf90/B of f0252adfc27b54a76a05585f3f45bf90 into cc7c51615aeb4494be1ea60782c5134a(size=12.8 K), total size for store is 12.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-24T22:14:05,945 DEBUG [RS:0;bd6592c37bb2:40087-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for f0252adfc27b54a76a05585f3f45bf90: 2024-11-24T22:14:05,945 INFO [RS:0;bd6592c37bb2:40087-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732486426439.f0252adfc27b54a76a05585f3f45bf90., storeName=f0252adfc27b54a76a05585f3f45bf90/B, priority=13, startTime=1732486445479; duration=0sec 2024-11-24T22:14:05,949 INFO [RS:0;bd6592c37bb2:40087-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in f0252adfc27b54a76a05585f3f45bf90/A of f0252adfc27b54a76a05585f3f45bf90 into da0dc8e8af434e93b77478eb2c1ab439(size=12.8 K), total size for store is 12.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-24T22:14:05,949 DEBUG [RS:0;bd6592c37bb2:40087-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for f0252adfc27b54a76a05585f3f45bf90: 2024-11-24T22:14:05,949 INFO [RS:0;bd6592c37bb2:40087-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732486426439.f0252adfc27b54a76a05585f3f45bf90., storeName=f0252adfc27b54a76a05585f3f45bf90/A, priority=13, startTime=1732486445478; duration=0sec 2024-11-24T22:14:05,949 DEBUG [RS:0;bd6592c37bb2:40087-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-24T22:14:05,949 DEBUG [RS:0;bd6592c37bb2:40087-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: f0252adfc27b54a76a05585f3f45bf90:A 2024-11-24T22:14:05,949 DEBUG [RS:0;bd6592c37bb2:40087-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-24T22:14:05,949 DEBUG [RS:0;bd6592c37bb2:40087-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-24T22:14:05,949 DEBUG [RS:0;bd6592c37bb2:40087-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: f0252adfc27b54a76a05585f3f45bf90:B 2024-11-24T22:14:05,951 DEBUG [RS:0;bd6592c37bb2:40087-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37585 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-24T22:14:05,951 DEBUG [RS:0;bd6592c37bb2:40087-shortCompactions-0 {}] regionserver.HStore(1541): f0252adfc27b54a76a05585f3f45bf90/C is initiating minor compaction (all files) 2024-11-24T22:14:05,951 INFO [RS:0;bd6592c37bb2:40087-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of f0252adfc27b54a76a05585f3f45bf90/C in TestAcidGuarantees,,1732486426439.f0252adfc27b54a76a05585f3f45bf90. 2024-11-24T22:14:05,951 INFO [RS:0;bd6592c37bb2:40087-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/f0252adfc27b54a76a05585f3f45bf90/C/05ada7a4ea8043d293e29bd7672e3405, hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/f0252adfc27b54a76a05585f3f45bf90/C/96d7872a2b26416798bfbc2e3cf792d4, hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/f0252adfc27b54a76a05585f3f45bf90/C/a92a99c7f4e04ab09097ec22773f3fd9] into tmpdir=hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/f0252adfc27b54a76a05585f3f45bf90/.tmp, totalSize=36.7 K 2024-11-24T22:14:05,952 DEBUG [RS:0;bd6592c37bb2:40087-shortCompactions-0 {}] compactions.Compactor(225): Compacting 05ada7a4ea8043d293e29bd7672e3405, keycount=150, bloomtype=ROW, size=12.7 K, encoding=NONE, compression=NONE, seqNum=512, earliestPutTs=1732486443305 2024-11-24T22:14:05,953 DEBUG [RS:0;bd6592c37bb2:40087-shortCompactions-0 {}] compactions.Compactor(225): Compacting 96d7872a2b26416798bfbc2e3cf792d4, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=548, earliestPutTs=1732486443438 2024-11-24T22:14:05,953 DEBUG [RS:0;bd6592c37bb2:40087-shortCompactions-0 {}] compactions.Compactor(225): Compacting a92a99c7f4e04ab09097ec22773f3fd9, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=562, earliestPutTs=1732486444123 2024-11-24T22:14:05,971 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37701 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=19 2024-11-24T22:14:05,977 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=40087 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=20 2024-11-24T22:14:05,977 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bd6592c37bb2:0-1 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732486426439.f0252adfc27b54a76a05585f3f45bf90. 2024-11-24T22:14:05,977 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bd6592c37bb2:0-1 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732486426439.f0252adfc27b54a76a05585f3f45bf90. as already flushing 2024-11-24T22:14:05,977 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bd6592c37bb2:0-1 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732486426439.f0252adfc27b54a76a05585f3f45bf90. 2024-11-24T22:14:05,977 ERROR [RS_FLUSH_OPERATIONS-regionserver/bd6592c37bb2:0-1 {event_type=RS_FLUSH_REGIONS, pid=20}] handler.RSProcedureHandler(58): pid=20 java.io.IOException: Unable to complete flush {ENCODED => f0252adfc27b54a76a05585f3f45bf90, NAME => 'TestAcidGuarantees,,1732486426439.f0252adfc27b54a76a05585f3f45bf90.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T22:14:05,977 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bd6592c37bb2:0-1 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=20 java.io.IOException: Unable to complete flush {ENCODED => f0252adfc27b54a76a05585f3f45bf90, NAME => 'TestAcidGuarantees,,1732486426439.f0252adfc27b54a76a05585f3f45bf90.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T22:14:05,980 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37701 {}] master.HMaster(4177): Remote procedure failed, pid=20 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => f0252adfc27b54a76a05585f3f45bf90, NAME => 'TestAcidGuarantees,,1732486426439.f0252adfc27b54a76a05585f3f45bf90.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => f0252adfc27b54a76a05585f3f45bf90, NAME => 'TestAcidGuarantees,,1732486426439.f0252adfc27b54a76a05585f3f45bf90.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T22:14:05,982 INFO [RS:0;bd6592c37bb2:40087-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): f0252adfc27b54a76a05585f3f45bf90#C#compaction#74 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-24T22:14:05,983 DEBUG [RS:0;bd6592c37bb2:40087-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/f0252adfc27b54a76a05585f3f45bf90/.tmp/C/8eb7820125494486bd2fd4f8fb01fe21 is 50, key is test_row_0/C:col10/1732486445267/Put/seqid=0 2024-11-24T22:14:06,014 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40087 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0252adfc27b54a76a05585f3f45bf90, server=bd6592c37bb2,40087,1732486423713 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-24T22:14:06,014 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40087 {}] ipc.CallRunner(138): callId: 207 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36532 deadline: 1732486506010, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0252adfc27b54a76a05585f3f45bf90, server=bd6592c37bb2,40087,1732486423713 2024-11-24T22:14:06,015 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40087 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0252adfc27b54a76a05585f3f45bf90, server=bd6592c37bb2,40087,1732486423713 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-24T22:14:06,015 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732486426439.f0252adfc27b54a76a05585f3f45bf90., hostname=bd6592c37bb2,40087,1732486423713, seqNum=2 , the old value is region=TestAcidGuarantees,,1732486426439.f0252adfc27b54a76a05585f3f45bf90., hostname=bd6592c37bb2,40087,1732486423713, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0252adfc27b54a76a05585f3f45bf90, server=bd6592c37bb2,40087,1732486423713 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T22:14:06,015 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732486426439.f0252adfc27b54a76a05585f3f45bf90., hostname=bd6592c37bb2,40087,1732486423713, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0252adfc27b54a76a05585f3f45bf90, server=bd6592c37bb2,40087,1732486423713 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T22:14:06,015 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40087 {}] ipc.CallRunner(138): callId: 233 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36518 deadline: 1732486506011, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0252adfc27b54a76a05585f3f45bf90, server=bd6592c37bb2,40087,1732486423713 2024-11-24T22:14:06,015 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732486426439.f0252adfc27b54a76a05585f3f45bf90., hostname=bd6592c37bb2,40087,1732486423713, seqNum=2 because the exception is null or not the one we care about 2024-11-24T22:14:06,015 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732486426439.f0252adfc27b54a76a05585f3f45bf90., hostname=bd6592c37bb2,40087,1732486423713, seqNum=2 , the old value is region=TestAcidGuarantees,,1732486426439.f0252adfc27b54a76a05585f3f45bf90., hostname=bd6592c37bb2,40087,1732486423713, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0252adfc27b54a76a05585f3f45bf90, server=bd6592c37bb2,40087,1732486423713 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T22:14:06,015 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732486426439.f0252adfc27b54a76a05585f3f45bf90., hostname=bd6592c37bb2,40087,1732486423713, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0252adfc27b54a76a05585f3f45bf90, server=bd6592c37bb2,40087,1732486423713 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T22:14:06,016 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732486426439.f0252adfc27b54a76a05585f3f45bf90., hostname=bd6592c37bb2,40087,1732486423713, seqNum=2 because the exception is null or not the one we care about 2024-11-24T22:14:06,040 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45929 is added to blk_1073741911_1087 (size=13085) 2024-11-24T22:14:06,133 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=40087 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=20 2024-11-24T22:14:06,136 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bd6592c37bb2:0-2 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732486426439.f0252adfc27b54a76a05585f3f45bf90. 2024-11-24T22:14:06,136 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bd6592c37bb2:0-2 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732486426439.f0252adfc27b54a76a05585f3f45bf90. as already flushing 2024-11-24T22:14:06,136 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bd6592c37bb2:0-2 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732486426439.f0252adfc27b54a76a05585f3f45bf90. 2024-11-24T22:14:06,136 ERROR [RS_FLUSH_OPERATIONS-regionserver/bd6592c37bb2:0-2 {event_type=RS_FLUSH_REGIONS, pid=20}] handler.RSProcedureHandler(58): pid=20 java.io.IOException: Unable to complete flush {ENCODED => f0252adfc27b54a76a05585f3f45bf90, NAME => 'TestAcidGuarantees,,1732486426439.f0252adfc27b54a76a05585f3f45bf90.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T22:14:06,136 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bd6592c37bb2:0-2 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=20 java.io.IOException: Unable to complete flush {ENCODED => f0252adfc27b54a76a05585f3f45bf90, NAME => 'TestAcidGuarantees,,1732486426439.f0252adfc27b54a76a05585f3f45bf90.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T22:14:06,137 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37701 {}] master.HMaster(4177): Remote procedure failed, pid=20 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => f0252adfc27b54a76a05585f3f45bf90, NAME => 'TestAcidGuarantees,,1732486426439.f0252adfc27b54a76a05585f3f45bf90.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => f0252adfc27b54a76a05585f3f45bf90, NAME => 'TestAcidGuarantees,,1732486426439.f0252adfc27b54a76a05585f3f45bf90.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T22:14:06,224 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=588 (bloomFilter=true), to=hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/f0252adfc27b54a76a05585f3f45bf90/.tmp/C/3e8950a3d04e473f9a791f9f71875f4e 2024-11-24T22:14:06,234 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40087 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0252adfc27b54a76a05585f3f45bf90, server=bd6592c37bb2,40087,1732486423713 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-24T22:14:06,235 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40087 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0252adfc27b54a76a05585f3f45bf90, server=bd6592c37bb2,40087,1732486423713 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-24T22:14:06,235 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40087 {}] ipc.CallRunner(138): callId: 234 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36518 deadline: 1732486506221, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0252adfc27b54a76a05585f3f45bf90, server=bd6592c37bb2,40087,1732486423713 2024-11-24T22:14:06,235 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40087 {}] ipc.CallRunner(138): callId: 208 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36532 deadline: 1732486506222, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0252adfc27b54a76a05585f3f45bf90, server=bd6592c37bb2,40087,1732486423713 2024-11-24T22:14:06,235 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732486426439.f0252adfc27b54a76a05585f3f45bf90., hostname=bd6592c37bb2,40087,1732486423713, seqNum=2 , the old value is region=TestAcidGuarantees,,1732486426439.f0252adfc27b54a76a05585f3f45bf90., hostname=bd6592c37bb2,40087,1732486423713, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0252adfc27b54a76a05585f3f45bf90, server=bd6592c37bb2,40087,1732486423713 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T22:14:06,235 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732486426439.f0252adfc27b54a76a05585f3f45bf90., hostname=bd6592c37bb2,40087,1732486423713, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0252adfc27b54a76a05585f3f45bf90, server=bd6592c37bb2,40087,1732486423713 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T22:14:06,236 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732486426439.f0252adfc27b54a76a05585f3f45bf90., hostname=bd6592c37bb2,40087,1732486423713, seqNum=2 because the exception is null or not the one we care about 2024-11-24T22:14:06,238 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732486426439.f0252adfc27b54a76a05585f3f45bf90., hostname=bd6592c37bb2,40087,1732486423713, seqNum=2 , the old value is region=TestAcidGuarantees,,1732486426439.f0252adfc27b54a76a05585f3f45bf90., hostname=bd6592c37bb2,40087,1732486423713, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0252adfc27b54a76a05585f3f45bf90, server=bd6592c37bb2,40087,1732486423713 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T22:14:06,238 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732486426439.f0252adfc27b54a76a05585f3f45bf90., hostname=bd6592c37bb2,40087,1732486423713, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0252adfc27b54a76a05585f3f45bf90, server=bd6592c37bb2,40087,1732486423713 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T22:14:06,238 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732486426439.f0252adfc27b54a76a05585f3f45bf90., hostname=bd6592c37bb2,40087,1732486423713, seqNum=2 because the exception is null or not the one we care about 2024-11-24T22:14:06,251 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/f0252adfc27b54a76a05585f3f45bf90/.tmp/A/bb4373777b7749fb9ffb570fede7c5e7 as hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/f0252adfc27b54a76a05585f3f45bf90/A/bb4373777b7749fb9ffb570fede7c5e7 2024-11-24T22:14:06,257 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/f0252adfc27b54a76a05585f3f45bf90/A/bb4373777b7749fb9ffb570fede7c5e7, entries=250, sequenceid=588, filesize=16.8 K 2024-11-24T22:14:06,258 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/f0252adfc27b54a76a05585f3f45bf90/.tmp/B/13d49dd09c36477fa2ccd8d4b6050657 as hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/f0252adfc27b54a76a05585f3f45bf90/B/13d49dd09c36477fa2ccd8d4b6050657 2024-11-24T22:14:06,265 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/f0252adfc27b54a76a05585f3f45bf90/B/13d49dd09c36477fa2ccd8d4b6050657, entries=150, sequenceid=588, filesize=12.0 K 2024-11-24T22:14:06,266 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/f0252adfc27b54a76a05585f3f45bf90/.tmp/C/3e8950a3d04e473f9a791f9f71875f4e as hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/f0252adfc27b54a76a05585f3f45bf90/C/3e8950a3d04e473f9a791f9f71875f4e 2024-11-24T22:14:06,272 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/f0252adfc27b54a76a05585f3f45bf90/C/3e8950a3d04e473f9a791f9f71875f4e, entries=150, sequenceid=588, filesize=12.0 K 2024-11-24T22:14:06,279 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~60.38 KB/61830, heapSize ~165.80 KB/169776, currentSize=125.23 KB/128240 for f0252adfc27b54a76a05585f3f45bf90 in 577ms, sequenceid=588, compaction requested=false 2024-11-24T22:14:06,279 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for f0252adfc27b54a76a05585f3f45bf90: 2024-11-24T22:14:06,281 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37701 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=19 2024-11-24T22:14:06,290 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=40087 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=20 2024-11-24T22:14:06,290 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bd6592c37bb2:0-0 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732486426439.f0252adfc27b54a76a05585f3f45bf90. 2024-11-24T22:14:06,291 INFO [RS_FLUSH_OPERATIONS-regionserver/bd6592c37bb2:0-0 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.HRegion(2902): Flushing f0252adfc27b54a76a05585f3f45bf90 3/3 column families, dataSize=125.23 KB heapSize=355.30 KB 2024-11-24T22:14:06,291 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bd6592c37bb2:0-0 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.CompactingMemStore(205): FLUSHING TO DISK f0252adfc27b54a76a05585f3f45bf90, store=A 2024-11-24T22:14:06,291 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bd6592c37bb2:0-0 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-24T22:14:06,291 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bd6592c37bb2:0-0 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.CompactingMemStore(205): FLUSHING TO DISK f0252adfc27b54a76a05585f3f45bf90, store=B 2024-11-24T22:14:06,291 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bd6592c37bb2:0-0 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-24T22:14:06,291 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bd6592c37bb2:0-0 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.CompactingMemStore(205): FLUSHING TO DISK f0252adfc27b54a76a05585f3f45bf90, store=C 2024-11-24T22:14:06,291 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bd6592c37bb2:0-0 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-24T22:14:06,311 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bd6592c37bb2:0-0 {event_type=RS_FLUSH_REGIONS, pid=20}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/f0252adfc27b54a76a05585f3f45bf90/.tmp/A/cfea60b26a5e4f2694b58c5df1e8f3d2 is 50, key is test_row_0/A:col10/1732486445893/Put/seqid=0 2024-11-24T22:14:06,335 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45929 is added to blk_1073741912_1088 (size=12301) 2024-11-24T22:14:06,337 INFO [RS_FLUSH_OPERATIONS-regionserver/bd6592c37bb2:0-0 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=625 (bloomFilter=true), to=hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/f0252adfc27b54a76a05585f3f45bf90/.tmp/A/cfea60b26a5e4f2694b58c5df1e8f3d2 2024-11-24T22:14:06,363 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bd6592c37bb2:0-0 {event_type=RS_FLUSH_REGIONS, pid=20}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/f0252adfc27b54a76a05585f3f45bf90/.tmp/B/60727be57d64464b947b404585a28958 is 50, key is test_row_0/B:col10/1732486445893/Put/seqid=0 2024-11-24T22:14:06,392 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45929 is added to blk_1073741913_1089 (size=12301) 2024-11-24T22:14:06,395 INFO [RS_FLUSH_OPERATIONS-regionserver/bd6592c37bb2:0-0 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=40.25 KB at sequenceid=625 (bloomFilter=true), to=hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/f0252adfc27b54a76a05585f3f45bf90/.tmp/B/60727be57d64464b947b404585a28958 2024-11-24T22:14:06,410 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bd6592c37bb2:0-0 {event_type=RS_FLUSH_REGIONS, pid=20}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/f0252adfc27b54a76a05585f3f45bf90/.tmp/C/6c8934fad0be457a99fbc833539d7a24 is 50, key is test_row_0/C:col10/1732486445893/Put/seqid=0 2024-11-24T22:14:06,426 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45929 is added to blk_1073741914_1090 (size=12301) 2024-11-24T22:14:06,429 INFO [RS_FLUSH_OPERATIONS-regionserver/bd6592c37bb2:0-0 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=40.25 KB at sequenceid=625 (bloomFilter=true), to=hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/f0252adfc27b54a76a05585f3f45bf90/.tmp/C/6c8934fad0be457a99fbc833539d7a24 2024-11-24T22:14:06,440 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bd6592c37bb2:0-0 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/f0252adfc27b54a76a05585f3f45bf90/.tmp/A/cfea60b26a5e4f2694b58c5df1e8f3d2 as hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/f0252adfc27b54a76a05585f3f45bf90/A/cfea60b26a5e4f2694b58c5df1e8f3d2 2024-11-24T22:14:06,447 INFO [RS_FLUSH_OPERATIONS-regionserver/bd6592c37bb2:0-0 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/f0252adfc27b54a76a05585f3f45bf90/A/cfea60b26a5e4f2694b58c5df1e8f3d2, entries=150, sequenceid=625, filesize=12.0 K 2024-11-24T22:14:06,448 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bd6592c37bb2:0-0 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/f0252adfc27b54a76a05585f3f45bf90/.tmp/B/60727be57d64464b947b404585a28958 as hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/f0252adfc27b54a76a05585f3f45bf90/B/60727be57d64464b947b404585a28958 2024-11-24T22:14:06,448 DEBUG [RS:0;bd6592c37bb2:40087-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/f0252adfc27b54a76a05585f3f45bf90/.tmp/C/8eb7820125494486bd2fd4f8fb01fe21 as hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/f0252adfc27b54a76a05585f3f45bf90/C/8eb7820125494486bd2fd4f8fb01fe21 2024-11-24T22:14:06,458 INFO [RS_FLUSH_OPERATIONS-regionserver/bd6592c37bb2:0-0 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/f0252adfc27b54a76a05585f3f45bf90/B/60727be57d64464b947b404585a28958, entries=150, sequenceid=625, filesize=12.0 K 2024-11-24T22:14:06,459 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bd6592c37bb2:0-0 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/f0252adfc27b54a76a05585f3f45bf90/.tmp/C/6c8934fad0be457a99fbc833539d7a24 as hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/f0252adfc27b54a76a05585f3f45bf90/C/6c8934fad0be457a99fbc833539d7a24 2024-11-24T22:14:06,460 INFO [RS:0;bd6592c37bb2:40087-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in f0252adfc27b54a76a05585f3f45bf90/C of f0252adfc27b54a76a05585f3f45bf90 into 8eb7820125494486bd2fd4f8fb01fe21(size=12.8 K), total size for store is 24.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-24T22:14:06,460 DEBUG [RS:0;bd6592c37bb2:40087-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for f0252adfc27b54a76a05585f3f45bf90: 2024-11-24T22:14:06,460 INFO [RS:0;bd6592c37bb2:40087-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732486426439.f0252adfc27b54a76a05585f3f45bf90., storeName=f0252adfc27b54a76a05585f3f45bf90/C, priority=13, startTime=1732486445479; duration=0sec 2024-11-24T22:14:06,460 DEBUG [RS:0;bd6592c37bb2:40087-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-24T22:14:06,460 DEBUG [RS:0;bd6592c37bb2:40087-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: f0252adfc27b54a76a05585f3f45bf90:C 2024-11-24T22:14:06,466 INFO [RS_FLUSH_OPERATIONS-regionserver/bd6592c37bb2:0-0 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/f0252adfc27b54a76a05585f3f45bf90/C/6c8934fad0be457a99fbc833539d7a24, entries=150, sequenceid=625, filesize=12.0 K 2024-11-24T22:14:06,467 INFO [RS_FLUSH_OPERATIONS-regionserver/bd6592c37bb2:0-0 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.HRegion(3140): Finished flush of dataSize ~125.23 KB/128240, heapSize ~355.25 KB/363776, currentSize=0 B/0 for f0252adfc27b54a76a05585f3f45bf90 in 176ms, sequenceid=625, compaction requested=true 2024-11-24T22:14:06,467 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bd6592c37bb2:0-0 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.HRegion(2603): Flush status journal for f0252adfc27b54a76a05585f3f45bf90: 2024-11-24T22:14:06,467 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bd6592c37bb2:0-0 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732486426439.f0252adfc27b54a76a05585f3f45bf90. 2024-11-24T22:14:06,468 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bd6592c37bb2:0-0 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=20 2024-11-24T22:14:06,468 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37701 {}] master.HMaster(4169): Remote procedure done, pid=20 2024-11-24T22:14:06,471 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=20, resume processing ppid=19 2024-11-24T22:14:06,471 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=20, ppid=19, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 810 msec 2024-11-24T22:14:06,474 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=19, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=19, table=TestAcidGuarantees in 820 msec 2024-11-24T22:14:06,552 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing f0252adfc27b54a76a05585f3f45bf90 3/3 column families, dataSize=53.67 KB heapSize=160.13 KB 2024-11-24T22:14:06,552 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK f0252adfc27b54a76a05585f3f45bf90, store=A 2024-11-24T22:14:06,552 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-24T22:14:06,552 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK f0252adfc27b54a76a05585f3f45bf90, store=B 2024-11-24T22:14:06,552 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-24T22:14:06,552 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK f0252adfc27b54a76a05585f3f45bf90, store=C 2024-11-24T22:14:06,552 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40087 {}] regionserver.HRegion(8855): Flush requested on f0252adfc27b54a76a05585f3f45bf90 2024-11-24T22:14:06,552 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-24T22:14:06,559 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/f0252adfc27b54a76a05585f3f45bf90/.tmp/A/efb9cfb73d5e4768bbda183d1331835c is 50, key is test_row_0/A:col10/1732486446550/Put/seqid=0 2024-11-24T22:14:06,571 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45929 is added to blk_1073741915_1091 (size=12301) 2024-11-24T22:14:06,609 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/bd6592c37bb2:0-3 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=309, cellSize=14145, totalHeapSize=32376, min timestamp=1732486446583, max timestamp=1732486446605Num uniques 859; 2024-11-24T22:14:06,609 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/bd6592c37bb2:0-1 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=309, cellSize=14145, totalHeapSize=32376, min timestamp=1732486446583, max timestamp=1732486446605Num uniques 859; 2024-11-24T22:14:06,609 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/bd6592c37bb2:0-3 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=14145, suffix heap size=117064,new segment heap size=32376 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=309 2024-11-24T22:14:06,609 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/bd6592c37bb2:0-1 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=14145, suffix heap size=117064,new segment heap size=32376 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=309 2024-11-24T22:14:06,609 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/bd6592c37bb2:0-5 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=309, cellSize=14145, totalHeapSize=32376, min timestamp=1732486446583, max timestamp=1732486446605Num uniques 859; 2024-11-24T22:14:06,609 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/bd6592c37bb2:0-5 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=14145, suffix heap size=117064,new segment heap size=32376 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=309 2024-11-24T22:14:06,658 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40087 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0252adfc27b54a76a05585f3f45bf90, server=bd6592c37bb2,40087,1732486423713 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-24T22:14:06,659 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40087 {}] ipc.CallRunner(138): callId: 254 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36518 deadline: 1732486506649, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0252adfc27b54a76a05585f3f45bf90, server=bd6592c37bb2,40087,1732486423713 2024-11-24T22:14:06,659 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732486426439.f0252adfc27b54a76a05585f3f45bf90., hostname=bd6592c37bb2,40087,1732486423713, seqNum=2 , the old value is region=TestAcidGuarantees,,1732486426439.f0252adfc27b54a76a05585f3f45bf90., hostname=bd6592c37bb2,40087,1732486423713, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0252adfc27b54a76a05585f3f45bf90, server=bd6592c37bb2,40087,1732486423713 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T22:14:06,659 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732486426439.f0252adfc27b54a76a05585f3f45bf90., hostname=bd6592c37bb2,40087,1732486423713, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0252adfc27b54a76a05585f3f45bf90, server=bd6592c37bb2,40087,1732486423713 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T22:14:06,659 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732486426439.f0252adfc27b54a76a05585f3f45bf90., hostname=bd6592c37bb2,40087,1732486423713, seqNum=2 because the exception is null or not the one we care about 2024-11-24T22:14:06,663 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40087 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0252adfc27b54a76a05585f3f45bf90, server=bd6592c37bb2,40087,1732486423713 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-24T22:14:06,663 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40087 {}] ipc.CallRunner(138): callId: 229 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36532 deadline: 1732486506658, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0252adfc27b54a76a05585f3f45bf90, server=bd6592c37bb2,40087,1732486423713 2024-11-24T22:14:06,663 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732486426439.f0252adfc27b54a76a05585f3f45bf90., hostname=bd6592c37bb2,40087,1732486423713, seqNum=2 , the old value is region=TestAcidGuarantees,,1732486426439.f0252adfc27b54a76a05585f3f45bf90., hostname=bd6592c37bb2,40087,1732486423713, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0252adfc27b54a76a05585f3f45bf90, server=bd6592c37bb2,40087,1732486423713 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T22:14:06,663 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732486426439.f0252adfc27b54a76a05585f3f45bf90., hostname=bd6592c37bb2,40087,1732486423713, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0252adfc27b54a76a05585f3f45bf90, server=bd6592c37bb2,40087,1732486423713 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T22:14:06,663 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732486426439.f0252adfc27b54a76a05585f3f45bf90., hostname=bd6592c37bb2,40087,1732486423713, seqNum=2 because the exception is null or not the one we care about 2024-11-24T22:14:06,764 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40087 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0252adfc27b54a76a05585f3f45bf90, server=bd6592c37bb2,40087,1732486423713 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-24T22:14:06,764 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40087 {}] ipc.CallRunner(138): callId: 255 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36518 deadline: 1732486506761, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0252adfc27b54a76a05585f3f45bf90, server=bd6592c37bb2,40087,1732486423713 2024-11-24T22:14:06,764 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732486426439.f0252adfc27b54a76a05585f3f45bf90., hostname=bd6592c37bb2,40087,1732486423713, seqNum=2 , the old value is region=TestAcidGuarantees,,1732486426439.f0252adfc27b54a76a05585f3f45bf90., hostname=bd6592c37bb2,40087,1732486423713, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0252adfc27b54a76a05585f3f45bf90, server=bd6592c37bb2,40087,1732486423713 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T22:14:06,764 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732486426439.f0252adfc27b54a76a05585f3f45bf90., hostname=bd6592c37bb2,40087,1732486423713, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0252adfc27b54a76a05585f3f45bf90, server=bd6592c37bb2,40087,1732486423713 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T22:14:06,764 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732486426439.f0252adfc27b54a76a05585f3f45bf90., hostname=bd6592c37bb2,40087,1732486423713, seqNum=2 because the exception is null or not the one we care about 2024-11-24T22:14:06,772 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40087 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0252adfc27b54a76a05585f3f45bf90, server=bd6592c37bb2,40087,1732486423713 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-24T22:14:06,773 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40087 {}] ipc.CallRunner(138): callId: 230 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36532 deadline: 1732486506771, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0252adfc27b54a76a05585f3f45bf90, server=bd6592c37bb2,40087,1732486423713 2024-11-24T22:14:06,773 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732486426439.f0252adfc27b54a76a05585f3f45bf90., hostname=bd6592c37bb2,40087,1732486423713, seqNum=2 , the old value is region=TestAcidGuarantees,,1732486426439.f0252adfc27b54a76a05585f3f45bf90., hostname=bd6592c37bb2,40087,1732486423713, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0252adfc27b54a76a05585f3f45bf90, server=bd6592c37bb2,40087,1732486423713 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T22:14:06,773 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732486426439.f0252adfc27b54a76a05585f3f45bf90., hostname=bd6592c37bb2,40087,1732486423713, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0252adfc27b54a76a05585f3f45bf90, server=bd6592c37bb2,40087,1732486423713 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T22:14:06,773 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732486426439.f0252adfc27b54a76a05585f3f45bf90., hostname=bd6592c37bb2,40087,1732486423713, seqNum=2 because the exception is null or not the one we care about 2024-11-24T22:14:06,791 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37701 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=19 2024-11-24T22:14:06,791 INFO [RPCClient-NioEventLoopGroup-4-8 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-24T22:14:06,793 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37701 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-24T22:14:06,794 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37701 {}] procedure2.ProcedureExecutor(1139): Stored pid=21, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=21, table=TestAcidGuarantees 2024-11-24T22:14:06,795 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37701 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=21 2024-11-24T22:14:06,795 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=21, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=21, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-24T22:14:06,796 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=21, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=21, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-24T22:14:06,796 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=22, ppid=21, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-24T22:14:06,901 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37701 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=21 2024-11-24T22:14:06,950 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=40087 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=22 2024-11-24T22:14:06,950 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bd6592c37bb2:0-1 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732486426439.f0252adfc27b54a76a05585f3f45bf90. 2024-11-24T22:14:06,950 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bd6592c37bb2:0-1 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732486426439.f0252adfc27b54a76a05585f3f45bf90. as already flushing 2024-11-24T22:14:06,951 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bd6592c37bb2:0-1 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732486426439.f0252adfc27b54a76a05585f3f45bf90. 2024-11-24T22:14:06,951 ERROR [RS_FLUSH_OPERATIONS-regionserver/bd6592c37bb2:0-1 {event_type=RS_FLUSH_REGIONS, pid=22}] handler.RSProcedureHandler(58): pid=22 java.io.IOException: Unable to complete flush {ENCODED => f0252adfc27b54a76a05585f3f45bf90, NAME => 'TestAcidGuarantees,,1732486426439.f0252adfc27b54a76a05585f3f45bf90.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T22:14:06,951 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bd6592c37bb2:0-1 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=22 java.io.IOException: Unable to complete flush {ENCODED => f0252adfc27b54a76a05585f3f45bf90, NAME => 'TestAcidGuarantees,,1732486426439.f0252adfc27b54a76a05585f3f45bf90.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T22:14:06,951 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37701 {}] master.HMaster(4177): Remote procedure failed, pid=22 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => f0252adfc27b54a76a05585f3f45bf90, NAME => 'TestAcidGuarantees,,1732486426439.f0252adfc27b54a76a05585f3f45bf90.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => f0252adfc27b54a76a05585f3f45bf90, NAME => 'TestAcidGuarantees,,1732486426439.f0252adfc27b54a76a05585f3f45bf90.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T22:14:06,969 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=637 (bloomFilter=true), to=hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/f0252adfc27b54a76a05585f3f45bf90/.tmp/A/efb9cfb73d5e4768bbda183d1331835c 2024-11-24T22:14:06,975 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40087 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0252adfc27b54a76a05585f3f45bf90, server=bd6592c37bb2,40087,1732486423713 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-24T22:14:06,975 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40087 {}] ipc.CallRunner(138): callId: 256 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36518 deadline: 1732486506971, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0252adfc27b54a76a05585f3f45bf90, server=bd6592c37bb2,40087,1732486423713 2024-11-24T22:14:06,976 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732486426439.f0252adfc27b54a76a05585f3f45bf90., hostname=bd6592c37bb2,40087,1732486423713, seqNum=2 , the old value is region=TestAcidGuarantees,,1732486426439.f0252adfc27b54a76a05585f3f45bf90., hostname=bd6592c37bb2,40087,1732486423713, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0252adfc27b54a76a05585f3f45bf90, server=bd6592c37bb2,40087,1732486423713 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T22:14:06,976 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732486426439.f0252adfc27b54a76a05585f3f45bf90., hostname=bd6592c37bb2,40087,1732486423713, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0252adfc27b54a76a05585f3f45bf90, server=bd6592c37bb2,40087,1732486423713 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T22:14:06,976 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732486426439.f0252adfc27b54a76a05585f3f45bf90., hostname=bd6592c37bb2,40087,1732486423713, seqNum=2 because the exception is null or not the one we care about 2024-11-24T22:14:06,985 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40087 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0252adfc27b54a76a05585f3f45bf90, server=bd6592c37bb2,40087,1732486423713 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-24T22:14:06,985 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40087 {}] ipc.CallRunner(138): callId: 231 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36532 deadline: 1732486506982, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0252adfc27b54a76a05585f3f45bf90, server=bd6592c37bb2,40087,1732486423713 2024-11-24T22:14:06,985 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732486426439.f0252adfc27b54a76a05585f3f45bf90., hostname=bd6592c37bb2,40087,1732486423713, seqNum=2 , the old value is region=TestAcidGuarantees,,1732486426439.f0252adfc27b54a76a05585f3f45bf90., hostname=bd6592c37bb2,40087,1732486423713, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0252adfc27b54a76a05585f3f45bf90, server=bd6592c37bb2,40087,1732486423713 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T22:14:06,986 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732486426439.f0252adfc27b54a76a05585f3f45bf90., hostname=bd6592c37bb2,40087,1732486423713, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0252adfc27b54a76a05585f3f45bf90, server=bd6592c37bb2,40087,1732486423713 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T22:14:06,986 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732486426439.f0252adfc27b54a76a05585f3f45bf90., hostname=bd6592c37bb2,40087,1732486423713, seqNum=2 because the exception is null or not the one we care about 2024-11-24T22:14:07,006 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/f0252adfc27b54a76a05585f3f45bf90/.tmp/B/81b92b86e15441e090171e98694d5519 is 50, key is test_row_0/B:col10/1732486446550/Put/seqid=0 2024-11-24T22:14:07,047 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45929 is added to blk_1073741916_1092 (size=12301) 2024-11-24T22:14:07,103 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=40087 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=22 2024-11-24T22:14:07,104 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bd6592c37bb2:0-2 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732486426439.f0252adfc27b54a76a05585f3f45bf90. 2024-11-24T22:14:07,104 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bd6592c37bb2:0-2 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732486426439.f0252adfc27b54a76a05585f3f45bf90. as already flushing 2024-11-24T22:14:07,104 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bd6592c37bb2:0-2 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732486426439.f0252adfc27b54a76a05585f3f45bf90. 2024-11-24T22:14:07,104 ERROR [RS_FLUSH_OPERATIONS-regionserver/bd6592c37bb2:0-2 {event_type=RS_FLUSH_REGIONS, pid=22}] handler.RSProcedureHandler(58): pid=22 java.io.IOException: Unable to complete flush {ENCODED => f0252adfc27b54a76a05585f3f45bf90, NAME => 'TestAcidGuarantees,,1732486426439.f0252adfc27b54a76a05585f3f45bf90.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T22:14:07,104 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bd6592c37bb2:0-2 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=22 java.io.IOException: Unable to complete flush {ENCODED => f0252adfc27b54a76a05585f3f45bf90, NAME => 'TestAcidGuarantees,,1732486426439.f0252adfc27b54a76a05585f3f45bf90.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T22:14:07,105 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37701 {}] master.HMaster(4177): Remote procedure failed, pid=22 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => f0252adfc27b54a76a05585f3f45bf90, NAME => 'TestAcidGuarantees,,1732486426439.f0252adfc27b54a76a05585f3f45bf90.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => f0252adfc27b54a76a05585f3f45bf90, NAME => 'TestAcidGuarantees,,1732486426439.f0252adfc27b54a76a05585f3f45bf90.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T22:14:07,110 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37701 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=21 2024-11-24T22:14:07,257 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=40087 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=22 2024-11-24T22:14:07,257 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bd6592c37bb2:0-0 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732486426439.f0252adfc27b54a76a05585f3f45bf90. 2024-11-24T22:14:07,257 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bd6592c37bb2:0-0 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732486426439.f0252adfc27b54a76a05585f3f45bf90. as already flushing 2024-11-24T22:14:07,257 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bd6592c37bb2:0-0 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732486426439.f0252adfc27b54a76a05585f3f45bf90. 2024-11-24T22:14:07,258 ERROR [RS_FLUSH_OPERATIONS-regionserver/bd6592c37bb2:0-0 {event_type=RS_FLUSH_REGIONS, pid=22}] handler.RSProcedureHandler(58): pid=22 java.io.IOException: Unable to complete flush {ENCODED => f0252adfc27b54a76a05585f3f45bf90, NAME => 'TestAcidGuarantees,,1732486426439.f0252adfc27b54a76a05585f3f45bf90.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T22:14:07,258 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bd6592c37bb2:0-0 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=22 java.io.IOException: Unable to complete flush {ENCODED => f0252adfc27b54a76a05585f3f45bf90, NAME => 'TestAcidGuarantees,,1732486426439.f0252adfc27b54a76a05585f3f45bf90.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T22:14:07,258 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37701 {}] master.HMaster(4177): Remote procedure failed, pid=22 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => f0252adfc27b54a76a05585f3f45bf90, NAME => 'TestAcidGuarantees,,1732486426439.f0252adfc27b54a76a05585f3f45bf90.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => f0252adfc27b54a76a05585f3f45bf90, NAME => 'TestAcidGuarantees,,1732486426439.f0252adfc27b54a76a05585f3f45bf90.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T22:14:07,286 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40087 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0252adfc27b54a76a05585f3f45bf90, server=bd6592c37bb2,40087,1732486423713 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-24T22:14:07,286 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40087 {}] ipc.CallRunner(138): callId: 257 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36518 deadline: 1732486507281, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0252adfc27b54a76a05585f3f45bf90, server=bd6592c37bb2,40087,1732486423713 2024-11-24T22:14:07,287 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732486426439.f0252adfc27b54a76a05585f3f45bf90., hostname=bd6592c37bb2,40087,1732486423713, seqNum=2 , the old value is region=TestAcidGuarantees,,1732486426439.f0252adfc27b54a76a05585f3f45bf90., hostname=bd6592c37bb2,40087,1732486423713, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0252adfc27b54a76a05585f3f45bf90, server=bd6592c37bb2,40087,1732486423713 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T22:14:07,287 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732486426439.f0252adfc27b54a76a05585f3f45bf90., hostname=bd6592c37bb2,40087,1732486423713, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0252adfc27b54a76a05585f3f45bf90, server=bd6592c37bb2,40087,1732486423713 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T22:14:07,287 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732486426439.f0252adfc27b54a76a05585f3f45bf90., hostname=bd6592c37bb2,40087,1732486423713, seqNum=2 because the exception is null or not the one we care about 2024-11-24T22:14:07,294 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40087 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0252adfc27b54a76a05585f3f45bf90, server=bd6592c37bb2,40087,1732486423713 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-24T22:14:07,294 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40087 {}] ipc.CallRunner(138): callId: 232 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36532 deadline: 1732486507291, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0252adfc27b54a76a05585f3f45bf90, server=bd6592c37bb2,40087,1732486423713 2024-11-24T22:14:07,297 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732486426439.f0252adfc27b54a76a05585f3f45bf90., hostname=bd6592c37bb2,40087,1732486423713, seqNum=2 , the old value is region=TestAcidGuarantees,,1732486426439.f0252adfc27b54a76a05585f3f45bf90., hostname=bd6592c37bb2,40087,1732486423713, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0252adfc27b54a76a05585f3f45bf90, server=bd6592c37bb2,40087,1732486423713 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T22:14:07,297 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732486426439.f0252adfc27b54a76a05585f3f45bf90., hostname=bd6592c37bb2,40087,1732486423713, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0252adfc27b54a76a05585f3f45bf90, server=bd6592c37bb2,40087,1732486423713 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T22:14:07,297 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732486426439.f0252adfc27b54a76a05585f3f45bf90., hostname=bd6592c37bb2,40087,1732486423713, seqNum=2 because the exception is null or not the one we care about 2024-11-24T22:14:07,411 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=40087 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=22 2024-11-24T22:14:07,411 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bd6592c37bb2:0-1 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732486426439.f0252adfc27b54a76a05585f3f45bf90. 2024-11-24T22:14:07,411 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bd6592c37bb2:0-1 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732486426439.f0252adfc27b54a76a05585f3f45bf90. as already flushing 2024-11-24T22:14:07,411 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bd6592c37bb2:0-1 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732486426439.f0252adfc27b54a76a05585f3f45bf90. 2024-11-24T22:14:07,411 ERROR [RS_FLUSH_OPERATIONS-regionserver/bd6592c37bb2:0-1 {event_type=RS_FLUSH_REGIONS, pid=22}] handler.RSProcedureHandler(58): pid=22 java.io.IOException: Unable to complete flush {ENCODED => f0252adfc27b54a76a05585f3f45bf90, NAME => 'TestAcidGuarantees,,1732486426439.f0252adfc27b54a76a05585f3f45bf90.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T22:14:07,412 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bd6592c37bb2:0-1 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=22 java.io.IOException: Unable to complete flush {ENCODED => f0252adfc27b54a76a05585f3f45bf90, NAME => 'TestAcidGuarantees,,1732486426439.f0252adfc27b54a76a05585f3f45bf90.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T22:14:07,412 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37701 {}] master.HMaster(4177): Remote procedure failed, pid=22 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => f0252adfc27b54a76a05585f3f45bf90, NAME => 'TestAcidGuarantees,,1732486426439.f0252adfc27b54a76a05585f3f45bf90.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => f0252adfc27b54a76a05585f3f45bf90, NAME => 'TestAcidGuarantees,,1732486426439.f0252adfc27b54a76a05585f3f45bf90.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T22:14:07,421 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37701 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=21 2024-11-24T22:14:07,448 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=637 (bloomFilter=true), to=hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/f0252adfc27b54a76a05585f3f45bf90/.tmp/B/81b92b86e15441e090171e98694d5519 2024-11-24T22:14:07,462 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/f0252adfc27b54a76a05585f3f45bf90/.tmp/C/017690ed58eb4d15a0a1ed73bc08a541 is 50, key is test_row_0/C:col10/1732486446550/Put/seqid=0 2024-11-24T22:14:07,474 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45929 is added to blk_1073741917_1093 (size=12301) 2024-11-24T22:14:07,475 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=637 (bloomFilter=true), to=hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/f0252adfc27b54a76a05585f3f45bf90/.tmp/C/017690ed58eb4d15a0a1ed73bc08a541 2024-11-24T22:14:07,482 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/f0252adfc27b54a76a05585f3f45bf90/.tmp/A/efb9cfb73d5e4768bbda183d1331835c as hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/f0252adfc27b54a76a05585f3f45bf90/A/efb9cfb73d5e4768bbda183d1331835c 2024-11-24T22:14:07,496 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/f0252adfc27b54a76a05585f3f45bf90/A/efb9cfb73d5e4768bbda183d1331835c, entries=150, sequenceid=637, filesize=12.0 K 2024-11-24T22:14:07,497 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/f0252adfc27b54a76a05585f3f45bf90/.tmp/B/81b92b86e15441e090171e98694d5519 as hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/f0252adfc27b54a76a05585f3f45bf90/B/81b92b86e15441e090171e98694d5519 2024-11-24T22:14:07,507 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/f0252adfc27b54a76a05585f3f45bf90/B/81b92b86e15441e090171e98694d5519, entries=150, sequenceid=637, filesize=12.0 K 2024-11-24T22:14:07,509 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/f0252adfc27b54a76a05585f3f45bf90/.tmp/C/017690ed58eb4d15a0a1ed73bc08a541 as hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/f0252adfc27b54a76a05585f3f45bf90/C/017690ed58eb4d15a0a1ed73bc08a541 2024-11-24T22:14:07,525 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/f0252adfc27b54a76a05585f3f45bf90/C/017690ed58eb4d15a0a1ed73bc08a541, entries=150, sequenceid=637, filesize=12.0 K 2024-11-24T22:14:07,526 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~53.67 KB/54960, heapSize ~160.08 KB/163920, currentSize=134.18 KB/137400 for f0252adfc27b54a76a05585f3f45bf90 in 975ms, sequenceid=637, compaction requested=true 2024-11-24T22:14:07,527 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for f0252adfc27b54a76a05585f3f45bf90: 2024-11-24T22:14:07,527 DEBUG [RS:0;bd6592c37bb2:40087-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-24T22:14:07,528 DEBUG [RS:0;bd6592c37bb2:40087-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 54868 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-24T22:14:07,529 DEBUG [RS:0;bd6592c37bb2:40087-shortCompactions-0 {}] regionserver.HStore(1541): f0252adfc27b54a76a05585f3f45bf90/A is initiating minor compaction (all files) 2024-11-24T22:14:07,529 INFO [RS:0;bd6592c37bb2:40087-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of f0252adfc27b54a76a05585f3f45bf90/A in TestAcidGuarantees,,1732486426439.f0252adfc27b54a76a05585f3f45bf90. 2024-11-24T22:14:07,529 INFO [RS:0;bd6592c37bb2:40087-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/f0252adfc27b54a76a05585f3f45bf90/A/da0dc8e8af434e93b77478eb2c1ab439, hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/f0252adfc27b54a76a05585f3f45bf90/A/bb4373777b7749fb9ffb570fede7c5e7, hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/f0252adfc27b54a76a05585f3f45bf90/A/cfea60b26a5e4f2694b58c5df1e8f3d2, hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/f0252adfc27b54a76a05585f3f45bf90/A/efb9cfb73d5e4768bbda183d1331835c] into tmpdir=hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/f0252adfc27b54a76a05585f3f45bf90/.tmp, totalSize=53.6 K 2024-11-24T22:14:07,529 DEBUG [RS:0;bd6592c37bb2:40087-shortCompactions-0 {}] compactions.Compactor(225): Compacting da0dc8e8af434e93b77478eb2c1ab439, keycount=150, bloomtype=ROW, size=12.8 K, encoding=NONE, compression=NONE, seqNum=562, earliestPutTs=1732486444123 2024-11-24T22:14:07,530 DEBUG [RS:0;bd6592c37bb2:40087-shortCompactions-0 {}] compactions.Compactor(225): Compacting bb4373777b7749fb9ffb570fede7c5e7, keycount=250, bloomtype=ROW, size=16.8 K, encoding=NONE, compression=NONE, seqNum=588, earliestPutTs=1732486445683 2024-11-24T22:14:07,530 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store f0252adfc27b54a76a05585f3f45bf90:A, priority=-2147483648, current under compaction store size is 1 2024-11-24T22:14:07,530 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-24T22:14:07,530 DEBUG [RS:0;bd6592c37bb2:40087-shortCompactions-0 {}] compactions.Compactor(225): Compacting cfea60b26a5e4f2694b58c5df1e8f3d2, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=625, earliestPutTs=1732486445872 2024-11-24T22:14:07,530 DEBUG [RS:0;bd6592c37bb2:40087-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-24T22:14:07,530 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store f0252adfc27b54a76a05585f3f45bf90:B, priority=-2147483648, current under compaction store size is 2 2024-11-24T22:14:07,531 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-24T22:14:07,531 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store f0252adfc27b54a76a05585f3f45bf90:C, priority=-2147483648, current under compaction store size is 3 2024-11-24T22:14:07,531 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-24T22:14:07,531 DEBUG [RS:0;bd6592c37bb2:40087-shortCompactions-0 {}] compactions.Compactor(225): Compacting efb9cfb73d5e4768bbda183d1331835c, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=637, earliestPutTs=1732486446547 2024-11-24T22:14:07,534 DEBUG [RS:0;bd6592c37bb2:40087-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 49988 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-24T22:14:07,534 DEBUG [RS:0;bd6592c37bb2:40087-longCompactions-0 {}] regionserver.HStore(1541): f0252adfc27b54a76a05585f3f45bf90/B is initiating minor compaction (all files) 2024-11-24T22:14:07,534 INFO [RS:0;bd6592c37bb2:40087-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of f0252adfc27b54a76a05585f3f45bf90/B in TestAcidGuarantees,,1732486426439.f0252adfc27b54a76a05585f3f45bf90. 2024-11-24T22:14:07,535 INFO [RS:0;bd6592c37bb2:40087-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/f0252adfc27b54a76a05585f3f45bf90/B/cc7c51615aeb4494be1ea60782c5134a, hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/f0252adfc27b54a76a05585f3f45bf90/B/13d49dd09c36477fa2ccd8d4b6050657, hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/f0252adfc27b54a76a05585f3f45bf90/B/60727be57d64464b947b404585a28958, hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/f0252adfc27b54a76a05585f3f45bf90/B/81b92b86e15441e090171e98694d5519] into tmpdir=hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/f0252adfc27b54a76a05585f3f45bf90/.tmp, totalSize=48.8 K 2024-11-24T22:14:07,535 DEBUG [RS:0;bd6592c37bb2:40087-longCompactions-0 {}] compactions.Compactor(225): Compacting cc7c51615aeb4494be1ea60782c5134a, keycount=150, bloomtype=ROW, size=12.8 K, encoding=NONE, compression=NONE, seqNum=562, earliestPutTs=1732486444123 2024-11-24T22:14:07,536 DEBUG [RS:0;bd6592c37bb2:40087-longCompactions-0 {}] compactions.Compactor(225): Compacting 13d49dd09c36477fa2ccd8d4b6050657, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=588, earliestPutTs=1732486445683 2024-11-24T22:14:07,537 DEBUG [RS:0;bd6592c37bb2:40087-longCompactions-0 {}] compactions.Compactor(225): Compacting 60727be57d64464b947b404585a28958, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=625, earliestPutTs=1732486445872 2024-11-24T22:14:07,538 DEBUG [RS:0;bd6592c37bb2:40087-longCompactions-0 {}] compactions.Compactor(225): Compacting 81b92b86e15441e090171e98694d5519, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=637, earliestPutTs=1732486446547 2024-11-24T22:14:07,561 INFO [RS:0;bd6592c37bb2:40087-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): f0252adfc27b54a76a05585f3f45bf90#A#compaction#81 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-24T22:14:07,562 DEBUG [RS:0;bd6592c37bb2:40087-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/f0252adfc27b54a76a05585f3f45bf90/.tmp/A/4b61696fbbe045e3b38c3abc9902e621 is 50, key is test_row_0/A:col10/1732486446550/Put/seqid=0 2024-11-24T22:14:07,566 INFO [RS:0;bd6592c37bb2:40087-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): f0252adfc27b54a76a05585f3f45bf90#B#compaction#82 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-24T22:14:07,567 DEBUG [RS:0;bd6592c37bb2:40087-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/f0252adfc27b54a76a05585f3f45bf90/.tmp/B/b390b104502d4d5393ffd3f302d7df00 is 50, key is test_row_0/B:col10/1732486446550/Put/seqid=0 2024-11-24T22:14:07,569 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=40087 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=22 2024-11-24T22:14:07,570 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bd6592c37bb2:0-2 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732486426439.f0252adfc27b54a76a05585f3f45bf90. 2024-11-24T22:14:07,570 INFO [RS_FLUSH_OPERATIONS-regionserver/bd6592c37bb2:0-2 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.HRegion(2902): Flushing f0252adfc27b54a76a05585f3f45bf90 3/3 column families, dataSize=134.18 KB heapSize=370.92 KB 2024-11-24T22:14:07,570 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bd6592c37bb2:0-2 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.CompactingMemStore(205): FLUSHING TO DISK f0252adfc27b54a76a05585f3f45bf90, store=A 2024-11-24T22:14:07,570 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bd6592c37bb2:0-2 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-24T22:14:07,570 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bd6592c37bb2:0-2 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.CompactingMemStore(205): FLUSHING TO DISK f0252adfc27b54a76a05585f3f45bf90, store=B 2024-11-24T22:14:07,571 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bd6592c37bb2:0-2 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-24T22:14:07,571 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bd6592c37bb2:0-2 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.CompactingMemStore(205): FLUSHING TO DISK f0252adfc27b54a76a05585f3f45bf90, store=C 2024-11-24T22:14:07,571 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bd6592c37bb2:0-2 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-24T22:14:07,591 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bd6592c37bb2:0-2 {event_type=RS_FLUSH_REGIONS, pid=22}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/f0252adfc27b54a76a05585f3f45bf90/.tmp/A/de916a9109584ecba9ec30586fe8366b is 50, key is test_row_0/A:col10/1732486446647/Put/seqid=0 2024-11-24T22:14:07,595 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45929 is added to blk_1073741918_1094 (size=13221) 2024-11-24T22:14:07,612 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45929 is added to blk_1073741919_1095 (size=13221) 2024-11-24T22:14:07,613 DEBUG [RS:0;bd6592c37bb2:40087-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/f0252adfc27b54a76a05585f3f45bf90/.tmp/A/4b61696fbbe045e3b38c3abc9902e621 as hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/f0252adfc27b54a76a05585f3f45bf90/A/4b61696fbbe045e3b38c3abc9902e621 2024-11-24T22:14:07,623 DEBUG [RS:0;bd6592c37bb2:40087-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/f0252adfc27b54a76a05585f3f45bf90/.tmp/B/b390b104502d4d5393ffd3f302d7df00 as hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/f0252adfc27b54a76a05585f3f45bf90/B/b390b104502d4d5393ffd3f302d7df00 2024-11-24T22:14:07,624 INFO [RS:0;bd6592c37bb2:40087-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in f0252adfc27b54a76a05585f3f45bf90/A of f0252adfc27b54a76a05585f3f45bf90 into 4b61696fbbe045e3b38c3abc9902e621(size=12.9 K), total size for store is 12.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-24T22:14:07,625 DEBUG [RS:0;bd6592c37bb2:40087-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for f0252adfc27b54a76a05585f3f45bf90: 2024-11-24T22:14:07,625 INFO [RS:0;bd6592c37bb2:40087-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732486426439.f0252adfc27b54a76a05585f3f45bf90., storeName=f0252adfc27b54a76a05585f3f45bf90/A, priority=12, startTime=1732486447527; duration=0sec 2024-11-24T22:14:07,625 DEBUG [RS:0;bd6592c37bb2:40087-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-24T22:14:07,625 DEBUG [RS:0;bd6592c37bb2:40087-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: f0252adfc27b54a76a05585f3f45bf90:A 2024-11-24T22:14:07,625 DEBUG [RS:0;bd6592c37bb2:40087-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-24T22:14:07,626 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45929 is added to blk_1073741920_1096 (size=12301) 2024-11-24T22:14:07,627 INFO [RS_FLUSH_OPERATIONS-regionserver/bd6592c37bb2:0-2 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=671 (bloomFilter=true), to=hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/f0252adfc27b54a76a05585f3f45bf90/.tmp/A/de916a9109584ecba9ec30586fe8366b 2024-11-24T22:14:07,628 DEBUG [RS:0;bd6592c37bb2:40087-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 49988 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-24T22:14:07,628 DEBUG [RS:0;bd6592c37bb2:40087-shortCompactions-0 {}] regionserver.HStore(1541): f0252adfc27b54a76a05585f3f45bf90/C is initiating minor compaction (all files) 2024-11-24T22:14:07,628 INFO [RS:0;bd6592c37bb2:40087-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of f0252adfc27b54a76a05585f3f45bf90/C in TestAcidGuarantees,,1732486426439.f0252adfc27b54a76a05585f3f45bf90. 2024-11-24T22:14:07,629 INFO [RS:0;bd6592c37bb2:40087-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/f0252adfc27b54a76a05585f3f45bf90/C/8eb7820125494486bd2fd4f8fb01fe21, hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/f0252adfc27b54a76a05585f3f45bf90/C/3e8950a3d04e473f9a791f9f71875f4e, hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/f0252adfc27b54a76a05585f3f45bf90/C/6c8934fad0be457a99fbc833539d7a24, hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/f0252adfc27b54a76a05585f3f45bf90/C/017690ed58eb4d15a0a1ed73bc08a541] into tmpdir=hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/f0252adfc27b54a76a05585f3f45bf90/.tmp, totalSize=48.8 K 2024-11-24T22:14:07,629 DEBUG [RS:0;bd6592c37bb2:40087-shortCompactions-0 {}] compactions.Compactor(225): Compacting 8eb7820125494486bd2fd4f8fb01fe21, keycount=150, bloomtype=ROW, size=12.8 K, encoding=NONE, compression=NONE, seqNum=562, earliestPutTs=1732486444123 2024-11-24T22:14:07,630 DEBUG [RS:0;bd6592c37bb2:40087-shortCompactions-0 {}] compactions.Compactor(225): Compacting 3e8950a3d04e473f9a791f9f71875f4e, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=588, earliestPutTs=1732486445683 2024-11-24T22:14:07,631 DEBUG [RS:0;bd6592c37bb2:40087-shortCompactions-0 {}] compactions.Compactor(225): Compacting 6c8934fad0be457a99fbc833539d7a24, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=625, earliestPutTs=1732486445872 2024-11-24T22:14:07,631 DEBUG [RS:0;bd6592c37bb2:40087-shortCompactions-0 {}] compactions.Compactor(225): Compacting 017690ed58eb4d15a0a1ed73bc08a541, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=637, earliestPutTs=1732486446547 2024-11-24T22:14:07,637 INFO [RS:0;bd6592c37bb2:40087-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in f0252adfc27b54a76a05585f3f45bf90/B of f0252adfc27b54a76a05585f3f45bf90 into b390b104502d4d5393ffd3f302d7df00(size=12.9 K), total size for store is 12.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-24T22:14:07,637 DEBUG [RS:0;bd6592c37bb2:40087-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for f0252adfc27b54a76a05585f3f45bf90: 2024-11-24T22:14:07,637 INFO [RS:0;bd6592c37bb2:40087-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732486426439.f0252adfc27b54a76a05585f3f45bf90., storeName=f0252adfc27b54a76a05585f3f45bf90/B, priority=12, startTime=1732486447530; duration=0sec 2024-11-24T22:14:07,638 DEBUG [RS:0;bd6592c37bb2:40087-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-24T22:14:07,638 DEBUG [RS:0;bd6592c37bb2:40087-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: f0252adfc27b54a76a05585f3f45bf90:B 2024-11-24T22:14:07,641 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bd6592c37bb2:0-2 {event_type=RS_FLUSH_REGIONS, pid=22}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/f0252adfc27b54a76a05585f3f45bf90/.tmp/B/1496c056f69e420c9011e49538e1d877 is 50, key is test_row_0/B:col10/1732486446647/Put/seqid=0 2024-11-24T22:14:07,645 INFO [RS:0;bd6592c37bb2:40087-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): f0252adfc27b54a76a05585f3f45bf90#C#compaction#85 average throughput is 2.18 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-24T22:14:07,646 DEBUG [RS:0;bd6592c37bb2:40087-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/f0252adfc27b54a76a05585f3f45bf90/.tmp/C/830623b4e53f462483bc52b3a55ecc37 is 50, key is test_row_0/C:col10/1732486446550/Put/seqid=0 2024-11-24T22:14:07,654 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45929 is added to blk_1073741921_1097 (size=12301) 2024-11-24T22:14:07,655 INFO [RS_FLUSH_OPERATIONS-regionserver/bd6592c37bb2:0-2 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=671 (bloomFilter=true), to=hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/f0252adfc27b54a76a05585f3f45bf90/.tmp/B/1496c056f69e420c9011e49538e1d877 2024-11-24T22:14:07,670 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bd6592c37bb2:0-2 {event_type=RS_FLUSH_REGIONS, pid=22}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/f0252adfc27b54a76a05585f3f45bf90/.tmp/C/94d5272cd82b4ec994e3c472dd9e27f2 is 50, key is test_row_0/C:col10/1732486446647/Put/seqid=0 2024-11-24T22:14:07,674 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45929 is added to blk_1073741922_1098 (size=13221) 2024-11-24T22:14:07,693 DEBUG [RS:0;bd6592c37bb2:40087-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/f0252adfc27b54a76a05585f3f45bf90/.tmp/C/830623b4e53f462483bc52b3a55ecc37 as hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/f0252adfc27b54a76a05585f3f45bf90/C/830623b4e53f462483bc52b3a55ecc37 2024-11-24T22:14:07,697 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45929 is added to blk_1073741923_1099 (size=12301) 2024-11-24T22:14:07,698 INFO [RS_FLUSH_OPERATIONS-regionserver/bd6592c37bb2:0-2 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=671 (bloomFilter=true), to=hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/f0252adfc27b54a76a05585f3f45bf90/.tmp/C/94d5272cd82b4ec994e3c472dd9e27f2 2024-11-24T22:14:07,705 INFO [RS:0;bd6592c37bb2:40087-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in f0252adfc27b54a76a05585f3f45bf90/C of f0252adfc27b54a76a05585f3f45bf90 into 830623b4e53f462483bc52b3a55ecc37(size=12.9 K), total size for store is 12.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-24T22:14:07,705 DEBUG [RS:0;bd6592c37bb2:40087-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for f0252adfc27b54a76a05585f3f45bf90: 2024-11-24T22:14:07,705 INFO [RS:0;bd6592c37bb2:40087-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732486426439.f0252adfc27b54a76a05585f3f45bf90., storeName=f0252adfc27b54a76a05585f3f45bf90/C, priority=12, startTime=1732486447531; duration=0sec 2024-11-24T22:14:07,705 DEBUG [RS:0;bd6592c37bb2:40087-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-24T22:14:07,705 DEBUG [RS:0;bd6592c37bb2:40087-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: f0252adfc27b54a76a05585f3f45bf90:C 2024-11-24T22:14:07,708 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bd6592c37bb2:0-2 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/f0252adfc27b54a76a05585f3f45bf90/.tmp/A/de916a9109584ecba9ec30586fe8366b as hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/f0252adfc27b54a76a05585f3f45bf90/A/de916a9109584ecba9ec30586fe8366b 2024-11-24T22:14:07,724 INFO [RS_FLUSH_OPERATIONS-regionserver/bd6592c37bb2:0-2 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/f0252adfc27b54a76a05585f3f45bf90/A/de916a9109584ecba9ec30586fe8366b, entries=150, sequenceid=671, filesize=12.0 K 2024-11-24T22:14:07,725 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bd6592c37bb2:0-2 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/f0252adfc27b54a76a05585f3f45bf90/.tmp/B/1496c056f69e420c9011e49538e1d877 as hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/f0252adfc27b54a76a05585f3f45bf90/B/1496c056f69e420c9011e49538e1d877 2024-11-24T22:14:07,733 INFO [RS_FLUSH_OPERATIONS-regionserver/bd6592c37bb2:0-2 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/f0252adfc27b54a76a05585f3f45bf90/B/1496c056f69e420c9011e49538e1d877, entries=150, sequenceid=671, filesize=12.0 K 2024-11-24T22:14:07,737 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bd6592c37bb2:0-2 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/f0252adfc27b54a76a05585f3f45bf90/.tmp/C/94d5272cd82b4ec994e3c472dd9e27f2 as hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/f0252adfc27b54a76a05585f3f45bf90/C/94d5272cd82b4ec994e3c472dd9e27f2 2024-11-24T22:14:07,746 INFO [RS_FLUSH_OPERATIONS-regionserver/bd6592c37bb2:0-2 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/f0252adfc27b54a76a05585f3f45bf90/C/94d5272cd82b4ec994e3c472dd9e27f2, entries=150, sequenceid=671, filesize=12.0 K 2024-11-24T22:14:07,747 INFO [RS_FLUSH_OPERATIONS-regionserver/bd6592c37bb2:0-2 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.HRegion(3140): Finished flush of dataSize ~134.18 KB/137400, heapSize ~370.88 KB/379776, currentSize=0 B/0 for f0252adfc27b54a76a05585f3f45bf90 in 177ms, sequenceid=671, compaction requested=false 2024-11-24T22:14:07,747 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bd6592c37bb2:0-2 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.HRegion(2603): Flush status journal for f0252adfc27b54a76a05585f3f45bf90: 2024-11-24T22:14:07,747 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bd6592c37bb2:0-2 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732486426439.f0252adfc27b54a76a05585f3f45bf90. 2024-11-24T22:14:07,748 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bd6592c37bb2:0-2 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=22 2024-11-24T22:14:07,748 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37701 {}] master.HMaster(4169): Remote procedure done, pid=22 2024-11-24T22:14:07,752 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=22, resume processing ppid=21 2024-11-24T22:14:07,752 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=22, ppid=21, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 953 msec 2024-11-24T22:14:07,754 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=21, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=21, table=TestAcidGuarantees in 960 msec 2024-11-24T22:14:07,816 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40087 {}] regionserver.HRegion(8855): Flush requested on f0252adfc27b54a76a05585f3f45bf90 2024-11-24T22:14:07,816 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing f0252adfc27b54a76a05585f3f45bf90 3/3 column families, dataSize=46.96 KB heapSize=140.20 KB 2024-11-24T22:14:07,816 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK f0252adfc27b54a76a05585f3f45bf90, store=A 2024-11-24T22:14:07,816 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-24T22:14:07,817 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK f0252adfc27b54a76a05585f3f45bf90, store=B 2024-11-24T22:14:07,817 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-24T22:14:07,817 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK f0252adfc27b54a76a05585f3f45bf90, store=C 2024-11-24T22:14:07,817 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-24T22:14:07,823 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/f0252adfc27b54a76a05585f3f45bf90/.tmp/A/2125dcf3fde74bf4bd202d0702d103aa is 50, key is test_row_0/A:col10/1732486447802/Put/seqid=0 2024-11-24T22:14:07,863 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45929 is added to blk_1073741924_1100 (size=12301) 2024-11-24T22:14:07,918 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/bd6592c37bb2:0-7 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=259, cellSize=11855, totalHeapSize=27176, min timestamp=1732486447893, max timestamp=1732486447914Num uniques 859; 2024-11-24T22:14:07,918 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/bd6592c37bb2:0-0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=259, cellSize=11855, totalHeapSize=27176, min timestamp=1732486447893, max timestamp=1732486447914Num uniques 859; 2024-11-24T22:14:07,918 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/bd6592c37bb2:0-7 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=11855, suffix heap size=117064,new segment heap size=27176 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=259 2024-11-24T22:14:07,919 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/bd6592c37bb2:0-0 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=11855, suffix heap size=117064,new segment heap size=27176 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=259 2024-11-24T22:14:07,919 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/bd6592c37bb2:0-4 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=259, cellSize=11855, totalHeapSize=27176, min timestamp=1732486447893, max timestamp=1732486447914Num uniques 859; 2024-11-24T22:14:07,919 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/bd6592c37bb2:0-4 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=11855, suffix heap size=117064,new segment heap size=27176 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=259 2024-11-24T22:14:07,931 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37701 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=21 2024-11-24T22:14:07,931 INFO [RPCClient-NioEventLoopGroup-4-8 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-24T22:14:07,932 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37701 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-24T22:14:07,933 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37701 {}] procedure2.ProcedureExecutor(1139): Stored pid=23, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=23, table=TestAcidGuarantees 2024-11-24T22:14:07,935 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37701 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=23 2024-11-24T22:14:07,936 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=23, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=23, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-24T22:14:07,937 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=23, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=23, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-24T22:14:07,937 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=24, ppid=23, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-24T22:14:07,981 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40087 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0252adfc27b54a76a05585f3f45bf90, server=bd6592c37bb2,40087,1732486423713 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-24T22:14:07,982 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40087 {}] ipc.CallRunner(138): callId: 251 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36532 deadline: 1732486507979, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0252adfc27b54a76a05585f3f45bf90, server=bd6592c37bb2,40087,1732486423713 2024-11-24T22:14:07,982 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732486426439.f0252adfc27b54a76a05585f3f45bf90., hostname=bd6592c37bb2,40087,1732486423713, seqNum=2 , the old value is region=TestAcidGuarantees,,1732486426439.f0252adfc27b54a76a05585f3f45bf90., hostname=bd6592c37bb2,40087,1732486423713, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0252adfc27b54a76a05585f3f45bf90, server=bd6592c37bb2,40087,1732486423713 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T22:14:07,982 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732486426439.f0252adfc27b54a76a05585f3f45bf90., hostname=bd6592c37bb2,40087,1732486423713, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0252adfc27b54a76a05585f3f45bf90, server=bd6592c37bb2,40087,1732486423713 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T22:14:07,982 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732486426439.f0252adfc27b54a76a05585f3f45bf90., hostname=bd6592c37bb2,40087,1732486423713, seqNum=2 because the exception is null or not the one we care about 2024-11-24T22:14:07,995 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40087 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0252adfc27b54a76a05585f3f45bf90, server=bd6592c37bb2,40087,1732486423713 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-24T22:14:07,996 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40087 {}] ipc.CallRunner(138): callId: 279 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36518 deadline: 1732486507982, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0252adfc27b54a76a05585f3f45bf90, server=bd6592c37bb2,40087,1732486423713 2024-11-24T22:14:07,996 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732486426439.f0252adfc27b54a76a05585f3f45bf90., hostname=bd6592c37bb2,40087,1732486423713, seqNum=2 , the old value is region=TestAcidGuarantees,,1732486426439.f0252adfc27b54a76a05585f3f45bf90., hostname=bd6592c37bb2,40087,1732486423713, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0252adfc27b54a76a05585f3f45bf90, server=bd6592c37bb2,40087,1732486423713 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T22:14:07,996 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732486426439.f0252adfc27b54a76a05585f3f45bf90., hostname=bd6592c37bb2,40087,1732486423713, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0252adfc27b54a76a05585f3f45bf90, server=bd6592c37bb2,40087,1732486423713 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T22:14:07,996 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732486426439.f0252adfc27b54a76a05585f3f45bf90., hostname=bd6592c37bb2,40087,1732486423713, seqNum=2 because the exception is null or not the one we care about 2024-11-24T22:14:08,041 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37701 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=23 2024-11-24T22:14:08,091 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=40087 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=24 2024-11-24T22:14:08,091 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bd6592c37bb2:0-0 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732486426439.f0252adfc27b54a76a05585f3f45bf90. 2024-11-24T22:14:08,091 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bd6592c37bb2:0-0 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732486426439.f0252adfc27b54a76a05585f3f45bf90. as already flushing 2024-11-24T22:14:08,091 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bd6592c37bb2:0-0 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732486426439.f0252adfc27b54a76a05585f3f45bf90. 2024-11-24T22:14:08,092 ERROR [RS_FLUSH_OPERATIONS-regionserver/bd6592c37bb2:0-0 {event_type=RS_FLUSH_REGIONS, pid=24}] handler.RSProcedureHandler(58): pid=24 java.io.IOException: Unable to complete flush {ENCODED => f0252adfc27b54a76a05585f3f45bf90, NAME => 'TestAcidGuarantees,,1732486426439.f0252adfc27b54a76a05585f3f45bf90.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T22:14:08,092 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bd6592c37bb2:0-0 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=24 java.io.IOException: Unable to complete flush {ENCODED => f0252adfc27b54a76a05585f3f45bf90, NAME => 'TestAcidGuarantees,,1732486426439.f0252adfc27b54a76a05585f3f45bf90.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T22:14:08,092 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37701 {}] master.HMaster(4177): Remote procedure failed, pid=24 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => f0252adfc27b54a76a05585f3f45bf90, NAME => 'TestAcidGuarantees,,1732486426439.f0252adfc27b54a76a05585f3f45bf90.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => f0252adfc27b54a76a05585f3f45bf90, NAME => 'TestAcidGuarantees,,1732486426439.f0252adfc27b54a76a05585f3f45bf90.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T22:14:08,096 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40087 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0252adfc27b54a76a05585f3f45bf90, server=bd6592c37bb2,40087,1732486423713 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-24T22:14:08,096 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40087 {}] ipc.CallRunner(138): callId: 252 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36532 deadline: 1732486508091, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0252adfc27b54a76a05585f3f45bf90, server=bd6592c37bb2,40087,1732486423713 2024-11-24T22:14:08,097 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732486426439.f0252adfc27b54a76a05585f3f45bf90., hostname=bd6592c37bb2,40087,1732486423713, seqNum=2 , the old value is region=TestAcidGuarantees,,1732486426439.f0252adfc27b54a76a05585f3f45bf90., hostname=bd6592c37bb2,40087,1732486423713, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0252adfc27b54a76a05585f3f45bf90, server=bd6592c37bb2,40087,1732486423713 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T22:14:08,097 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732486426439.f0252adfc27b54a76a05585f3f45bf90., hostname=bd6592c37bb2,40087,1732486423713, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0252adfc27b54a76a05585f3f45bf90, server=bd6592c37bb2,40087,1732486423713 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T22:14:08,097 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732486426439.f0252adfc27b54a76a05585f3f45bf90., hostname=bd6592c37bb2,40087,1732486423713, seqNum=2 because the exception is null or not the one we care about 2024-11-24T22:14:08,107 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40087 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0252adfc27b54a76a05585f3f45bf90, server=bd6592c37bb2,40087,1732486423713 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-24T22:14:08,108 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40087 {}] ipc.CallRunner(138): callId: 280 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36518 deadline: 1732486508107, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0252adfc27b54a76a05585f3f45bf90, server=bd6592c37bb2,40087,1732486423713 2024-11-24T22:14:08,108 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732486426439.f0252adfc27b54a76a05585f3f45bf90., hostname=bd6592c37bb2,40087,1732486423713, seqNum=2 , the old value is region=TestAcidGuarantees,,1732486426439.f0252adfc27b54a76a05585f3f45bf90., hostname=bd6592c37bb2,40087,1732486423713, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0252adfc27b54a76a05585f3f45bf90, server=bd6592c37bb2,40087,1732486423713 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T22:14:08,108 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732486426439.f0252adfc27b54a76a05585f3f45bf90., hostname=bd6592c37bb2,40087,1732486423713, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0252adfc27b54a76a05585f3f45bf90, server=bd6592c37bb2,40087,1732486423713 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T22:14:08,108 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732486426439.f0252adfc27b54a76a05585f3f45bf90., hostname=bd6592c37bb2,40087,1732486423713, seqNum=2 because the exception is null or not the one we care about 2024-11-24T22:14:08,235 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40087 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0252adfc27b54a76a05585f3f45bf90, server=bd6592c37bb2,40087,1732486423713 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-24T22:14:08,236 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40087 {}] ipc.CallRunner(138): callId: 102 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36548 deadline: 1732486508231, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0252adfc27b54a76a05585f3f45bf90, server=bd6592c37bb2,40087,1732486423713 2024-11-24T22:14:08,236 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40087 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0252adfc27b54a76a05585f3f45bf90, server=bd6592c37bb2,40087,1732486423713 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-24T22:14:08,236 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40087 {}] ipc.CallRunner(138): callId: 121 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36562 deadline: 1732486508231, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0252adfc27b54a76a05585f3f45bf90, server=bd6592c37bb2,40087,1732486423713 2024-11-24T22:14:08,236 WARN [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRpcRetryingCaller(177): Call to bd6592c37bb2,40087,1732486423713 for 'test_row_0' in f0252adfc27b54a76a05585f3f45bf90 of TestAcidGuarantees failed, tries = 8, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 8233 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0252adfc27b54a76a05585f3f45bf90, server=bd6592c37bb2,40087,1732486423713 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor44.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0252adfc27b54a76a05585f3f45bf90, server=bd6592c37bb2,40087,1732486423713 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-24T22:14:08,237 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732486426439.f0252adfc27b54a76a05585f3f45bf90., hostname=bd6592c37bb2,40087,1732486423713, seqNum=2 , the old value is region=TestAcidGuarantees,,1732486426439.f0252adfc27b54a76a05585f3f45bf90., hostname=bd6592c37bb2,40087,1732486423713, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0252adfc27b54a76a05585f3f45bf90, server=bd6592c37bb2,40087,1732486423713 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T22:14:08,237 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732486426439.f0252adfc27b54a76a05585f3f45bf90., hostname=bd6592c37bb2,40087,1732486423713, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0252adfc27b54a76a05585f3f45bf90, server=bd6592c37bb2,40087,1732486423713 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T22:14:08,237 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732486426439.f0252adfc27b54a76a05585f3f45bf90., hostname=bd6592c37bb2,40087,1732486423713, seqNum=2 because the exception is null or not the one we care about 2024-11-24T22:14:08,237 WARN [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRpcRetryingCaller(177): Call to bd6592c37bb2,40087,1732486423713 for 'test_row_0' in f0252adfc27b54a76a05585f3f45bf90 of TestAcidGuarantees failed, tries = 8, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 8233 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0252adfc27b54a76a05585f3f45bf90, server=bd6592c37bb2,40087,1732486423713 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor44.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0252adfc27b54a76a05585f3f45bf90, server=bd6592c37bb2,40087,1732486423713 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-24T22:14:08,238 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732486426439.f0252adfc27b54a76a05585f3f45bf90., hostname=bd6592c37bb2,40087,1732486423713, seqNum=2 , the old value is region=TestAcidGuarantees,,1732486426439.f0252adfc27b54a76a05585f3f45bf90., hostname=bd6592c37bb2,40087,1732486423713, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0252adfc27b54a76a05585f3f45bf90, server=bd6592c37bb2,40087,1732486423713 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T22:14:08,238 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732486426439.f0252adfc27b54a76a05585f3f45bf90., hostname=bd6592c37bb2,40087,1732486423713, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0252adfc27b54a76a05585f3f45bf90, server=bd6592c37bb2,40087,1732486423713 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T22:14:08,238 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732486426439.f0252adfc27b54a76a05585f3f45bf90., hostname=bd6592c37bb2,40087,1732486423713, seqNum=2 because the exception is null or not the one we care about 2024-11-24T22:14:08,244 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40087 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0252adfc27b54a76a05585f3f45bf90, server=bd6592c37bb2,40087,1732486423713 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-24T22:14:08,244 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40087 {}] ipc.CallRunner(138): callId: 108 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36590 deadline: 1732486508241, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0252adfc27b54a76a05585f3f45bf90, server=bd6592c37bb2,40087,1732486423713 2024-11-24T22:14:08,245 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=40087 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=24 2024-11-24T22:14:08,245 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bd6592c37bb2:0-1 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732486426439.f0252adfc27b54a76a05585f3f45bf90. 2024-11-24T22:14:08,246 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bd6592c37bb2:0-1 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732486426439.f0252adfc27b54a76a05585f3f45bf90. as already flushing 2024-11-24T22:14:08,246 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bd6592c37bb2:0-1 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732486426439.f0252adfc27b54a76a05585f3f45bf90. 2024-11-24T22:14:08,246 ERROR [RS_FLUSH_OPERATIONS-regionserver/bd6592c37bb2:0-1 {event_type=RS_FLUSH_REGIONS, pid=24}] handler.RSProcedureHandler(58): pid=24 java.io.IOException: Unable to complete flush {ENCODED => f0252adfc27b54a76a05585f3f45bf90, NAME => 'TestAcidGuarantees,,1732486426439.f0252adfc27b54a76a05585f3f45bf90.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T22:14:08,245 WARN [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRpcRetryingCaller(177): Call to bd6592c37bb2,40087,1732486423713 for 'test_row_1' in f0252adfc27b54a76a05585f3f45bf90 of TestAcidGuarantees failed, tries = 8, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 8246 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0252adfc27b54a76a05585f3f45bf90, server=bd6592c37bb2,40087,1732486423713 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor44.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0252adfc27b54a76a05585f3f45bf90, server=bd6592c37bb2,40087,1732486423713 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-24T22:14:08,246 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bd6592c37bb2:0-1 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=24 java.io.IOException: Unable to complete flush {ENCODED => f0252adfc27b54a76a05585f3f45bf90, NAME => 'TestAcidGuarantees,,1732486426439.f0252adfc27b54a76a05585f3f45bf90.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T22:14:08,246 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732486426439.f0252adfc27b54a76a05585f3f45bf90., hostname=bd6592c37bb2,40087,1732486423713, seqNum=2 , the old value is region=TestAcidGuarantees,,1732486426439.f0252adfc27b54a76a05585f3f45bf90., hostname=bd6592c37bb2,40087,1732486423713, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0252adfc27b54a76a05585f3f45bf90, server=bd6592c37bb2,40087,1732486423713 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T22:14:08,246 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732486426439.f0252adfc27b54a76a05585f3f45bf90., hostname=bd6592c37bb2,40087,1732486423713, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0252adfc27b54a76a05585f3f45bf90, server=bd6592c37bb2,40087,1732486423713 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T22:14:08,246 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732486426439.f0252adfc27b54a76a05585f3f45bf90., hostname=bd6592c37bb2,40087,1732486423713, seqNum=2 because the exception is null or not the one we care about 2024-11-24T22:14:08,247 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37701 {}] master.HMaster(4177): Remote procedure failed, pid=24 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => f0252adfc27b54a76a05585f3f45bf90, NAME => 'TestAcidGuarantees,,1732486426439.f0252adfc27b54a76a05585f3f45bf90.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => f0252adfc27b54a76a05585f3f45bf90, NAME => 'TestAcidGuarantees,,1732486426439.f0252adfc27b54a76a05585f3f45bf90.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T22:14:08,252 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37701 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=23 2024-11-24T22:14:08,265 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=684 (bloomFilter=true), to=hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/f0252adfc27b54a76a05585f3f45bf90/.tmp/A/2125dcf3fde74bf4bd202d0702d103aa 2024-11-24T22:14:08,280 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/f0252adfc27b54a76a05585f3f45bf90/.tmp/B/6662a3106960477e8a8e7608436590b0 is 50, key is test_row_0/B:col10/1732486447802/Put/seqid=0 2024-11-24T22:14:08,286 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45929 is added to blk_1073741925_1101 (size=12301) 2024-11-24T22:14:08,307 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40087 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0252adfc27b54a76a05585f3f45bf90, server=bd6592c37bb2,40087,1732486423713 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-24T22:14:08,307 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40087 {}] ipc.CallRunner(138): callId: 253 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36532 deadline: 1732486508301, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0252adfc27b54a76a05585f3f45bf90, server=bd6592c37bb2,40087,1732486423713 2024-11-24T22:14:08,307 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732486426439.f0252adfc27b54a76a05585f3f45bf90., hostname=bd6592c37bb2,40087,1732486423713, seqNum=2 , the old value is region=TestAcidGuarantees,,1732486426439.f0252adfc27b54a76a05585f3f45bf90., hostname=bd6592c37bb2,40087,1732486423713, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0252adfc27b54a76a05585f3f45bf90, server=bd6592c37bb2,40087,1732486423713 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T22:14:08,308 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732486426439.f0252adfc27b54a76a05585f3f45bf90., hostname=bd6592c37bb2,40087,1732486423713, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0252adfc27b54a76a05585f3f45bf90, server=bd6592c37bb2,40087,1732486423713 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T22:14:08,308 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732486426439.f0252adfc27b54a76a05585f3f45bf90., hostname=bd6592c37bb2,40087,1732486423713, seqNum=2 because the exception is null or not the one we care about 2024-11-24T22:14:08,315 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40087 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0252adfc27b54a76a05585f3f45bf90, server=bd6592c37bb2,40087,1732486423713 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-24T22:14:08,315 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40087 {}] ipc.CallRunner(138): callId: 281 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36518 deadline: 1732486508310, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0252adfc27b54a76a05585f3f45bf90, server=bd6592c37bb2,40087,1732486423713 2024-11-24T22:14:08,315 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732486426439.f0252adfc27b54a76a05585f3f45bf90., hostname=bd6592c37bb2,40087,1732486423713, seqNum=2 , the old value is region=TestAcidGuarantees,,1732486426439.f0252adfc27b54a76a05585f3f45bf90., hostname=bd6592c37bb2,40087,1732486423713, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0252adfc27b54a76a05585f3f45bf90, server=bd6592c37bb2,40087,1732486423713 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T22:14:08,316 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732486426439.f0252adfc27b54a76a05585f3f45bf90., hostname=bd6592c37bb2,40087,1732486423713, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0252adfc27b54a76a05585f3f45bf90, server=bd6592c37bb2,40087,1732486423713 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T22:14:08,316 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732486426439.f0252adfc27b54a76a05585f3f45bf90., hostname=bd6592c37bb2,40087,1732486423713, seqNum=2 because the exception is null or not the one we care about 2024-11-24T22:14:08,400 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=40087 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=24 2024-11-24T22:14:08,401 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bd6592c37bb2:0-2 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732486426439.f0252adfc27b54a76a05585f3f45bf90. 2024-11-24T22:14:08,401 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bd6592c37bb2:0-2 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732486426439.f0252adfc27b54a76a05585f3f45bf90. as already flushing 2024-11-24T22:14:08,401 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bd6592c37bb2:0-2 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732486426439.f0252adfc27b54a76a05585f3f45bf90. 2024-11-24T22:14:08,401 ERROR [RS_FLUSH_OPERATIONS-regionserver/bd6592c37bb2:0-2 {event_type=RS_FLUSH_REGIONS, pid=24}] handler.RSProcedureHandler(58): pid=24 java.io.IOException: Unable to complete flush {ENCODED => f0252adfc27b54a76a05585f3f45bf90, NAME => 'TestAcidGuarantees,,1732486426439.f0252adfc27b54a76a05585f3f45bf90.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T22:14:08,401 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bd6592c37bb2:0-2 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=24 java.io.IOException: Unable to complete flush {ENCODED => f0252adfc27b54a76a05585f3f45bf90, NAME => 'TestAcidGuarantees,,1732486426439.f0252adfc27b54a76a05585f3f45bf90.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T22:14:08,402 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37701 {}] master.HMaster(4177): Remote procedure failed, pid=24 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => f0252adfc27b54a76a05585f3f45bf90, NAME => 'TestAcidGuarantees,,1732486426439.f0252adfc27b54a76a05585f3f45bf90.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => f0252adfc27b54a76a05585f3f45bf90, NAME => 'TestAcidGuarantees,,1732486426439.f0252adfc27b54a76a05585f3f45bf90.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T22:14:08,554 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=40087 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=24 2024-11-24T22:14:08,554 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bd6592c37bb2:0-0 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732486426439.f0252adfc27b54a76a05585f3f45bf90. 2024-11-24T22:14:08,554 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bd6592c37bb2:0-0 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732486426439.f0252adfc27b54a76a05585f3f45bf90. as already flushing 2024-11-24T22:14:08,554 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bd6592c37bb2:0-0 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732486426439.f0252adfc27b54a76a05585f3f45bf90. 2024-11-24T22:14:08,554 ERROR [RS_FLUSH_OPERATIONS-regionserver/bd6592c37bb2:0-0 {event_type=RS_FLUSH_REGIONS, pid=24}] handler.RSProcedureHandler(58): pid=24 java.io.IOException: Unable to complete flush {ENCODED => f0252adfc27b54a76a05585f3f45bf90, NAME => 'TestAcidGuarantees,,1732486426439.f0252adfc27b54a76a05585f3f45bf90.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T22:14:08,555 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bd6592c37bb2:0-0 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=24 java.io.IOException: Unable to complete flush {ENCODED => f0252adfc27b54a76a05585f3f45bf90, NAME => 'TestAcidGuarantees,,1732486426439.f0252adfc27b54a76a05585f3f45bf90.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T22:14:08,555 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37701 {}] master.HMaster(4177): Remote procedure failed, pid=24 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => f0252adfc27b54a76a05585f3f45bf90, NAME => 'TestAcidGuarantees,,1732486426439.f0252adfc27b54a76a05585f3f45bf90.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => f0252adfc27b54a76a05585f3f45bf90, NAME => 'TestAcidGuarantees,,1732486426439.f0252adfc27b54a76a05585f3f45bf90.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T22:14:08,561 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37701 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=23 2024-11-24T22:14:08,616 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40087 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0252adfc27b54a76a05585f3f45bf90, server=bd6592c37bb2,40087,1732486423713 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-24T22:14:08,616 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40087 {}] ipc.CallRunner(138): callId: 254 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36532 deadline: 1732486508612, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0252adfc27b54a76a05585f3f45bf90, server=bd6592c37bb2,40087,1732486423713 2024-11-24T22:14:08,617 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732486426439.f0252adfc27b54a76a05585f3f45bf90., hostname=bd6592c37bb2,40087,1732486423713, seqNum=2 , the old value is region=TestAcidGuarantees,,1732486426439.f0252adfc27b54a76a05585f3f45bf90., hostname=bd6592c37bb2,40087,1732486423713, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0252adfc27b54a76a05585f3f45bf90, server=bd6592c37bb2,40087,1732486423713 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T22:14:08,617 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732486426439.f0252adfc27b54a76a05585f3f45bf90., hostname=bd6592c37bb2,40087,1732486423713, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0252adfc27b54a76a05585f3f45bf90, server=bd6592c37bb2,40087,1732486423713 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T22:14:08,617 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732486426439.f0252adfc27b54a76a05585f3f45bf90., hostname=bd6592c37bb2,40087,1732486423713, seqNum=2 because the exception is null or not the one we care about 2024-11-24T22:14:08,624 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40087 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0252adfc27b54a76a05585f3f45bf90, server=bd6592c37bb2,40087,1732486423713 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-24T22:14:08,624 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40087 {}] ipc.CallRunner(138): callId: 282 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36518 deadline: 1732486508621, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0252adfc27b54a76a05585f3f45bf90, server=bd6592c37bb2,40087,1732486423713 2024-11-24T22:14:08,625 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732486426439.f0252adfc27b54a76a05585f3f45bf90., hostname=bd6592c37bb2,40087,1732486423713, seqNum=2 , the old value is region=TestAcidGuarantees,,1732486426439.f0252adfc27b54a76a05585f3f45bf90., hostname=bd6592c37bb2,40087,1732486423713, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0252adfc27b54a76a05585f3f45bf90, server=bd6592c37bb2,40087,1732486423713 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T22:14:08,625 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732486426439.f0252adfc27b54a76a05585f3f45bf90., hostname=bd6592c37bb2,40087,1732486423713, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0252adfc27b54a76a05585f3f45bf90, server=bd6592c37bb2,40087,1732486423713 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T22:14:08,625 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732486426439.f0252adfc27b54a76a05585f3f45bf90., hostname=bd6592c37bb2,40087,1732486423713, seqNum=2 because the exception is null or not the one we care about 2024-11-24T22:14:08,687 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=684 (bloomFilter=true), to=hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/f0252adfc27b54a76a05585f3f45bf90/.tmp/B/6662a3106960477e8a8e7608436590b0 2024-11-24T22:14:08,698 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/f0252adfc27b54a76a05585f3f45bf90/.tmp/C/3fec91296d704444acf8427c12e1b565 is 50, key is test_row_0/C:col10/1732486447802/Put/seqid=0 2024-11-24T22:14:08,702 INFO [Thread-154 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-154. 2024-11-24T22:14:08,702 DEBUG [Thread-154 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicGetReader.workDone(AcidGuaranteesTestTool.java:234) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-24T22:14:08,705 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45929 is added to blk_1073741926_1102 (size=12301) 2024-11-24T22:14:08,708 DEBUG [Thread-154 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-24T22:14:08,709 DEBUG [Thread-154 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-24T22:14:08,709 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-24T22:14:08,709 INFO [Thread-156 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-156. 2024-11-24T22:14:08,709 DEBUG [Thread-156 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicGetReader.workDone(AcidGuaranteesTestTool.java:234) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-24T22:14:08,709 INFO [Thread-160 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-160. 2024-11-24T22:14:08,709 DEBUG [Thread-156 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-24T22:14:08,709 DEBUG [Thread-160 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicScanReader.workDone(AcidGuaranteesTestTool.java:302) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-24T22:14:08,709 DEBUG [Thread-160 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-24T22:14:08,709 DEBUG [Thread-156 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-24T22:14:08,709 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-24T22:14:08,709 DEBUG [Thread-160 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-24T22:14:08,710 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-24T22:14:08,710 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=40087 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=24 2024-11-24T22:14:08,710 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bd6592c37bb2:0-1 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732486426439.f0252adfc27b54a76a05585f3f45bf90. 2024-11-24T22:14:08,710 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bd6592c37bb2:0-1 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732486426439.f0252adfc27b54a76a05585f3f45bf90. as already flushing 2024-11-24T22:14:08,710 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bd6592c37bb2:0-1 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732486426439.f0252adfc27b54a76a05585f3f45bf90. 2024-11-24T22:14:08,710 ERROR [RS_FLUSH_OPERATIONS-regionserver/bd6592c37bb2:0-1 {event_type=RS_FLUSH_REGIONS, pid=24}] handler.RSProcedureHandler(58): pid=24 java.io.IOException: Unable to complete flush {ENCODED => f0252adfc27b54a76a05585f3f45bf90, NAME => 'TestAcidGuarantees,,1732486426439.f0252adfc27b54a76a05585f3f45bf90.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T22:14:08,710 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bd6592c37bb2:0-1 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=24 java.io.IOException: Unable to complete flush {ENCODED => f0252adfc27b54a76a05585f3f45bf90, NAME => 'TestAcidGuarantees,,1732486426439.f0252adfc27b54a76a05585f3f45bf90.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T22:14:08,712 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37701 {}] master.HMaster(4177): Remote procedure failed, pid=24 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => f0252adfc27b54a76a05585f3f45bf90, NAME => 'TestAcidGuarantees,,1732486426439.f0252adfc27b54a76a05585f3f45bf90.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => f0252adfc27b54a76a05585f3f45bf90, NAME => 'TestAcidGuarantees,,1732486426439.f0252adfc27b54a76a05585f3f45bf90.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T22:14:08,717 INFO [Thread-158 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-158. 2024-11-24T22:14:08,717 DEBUG [Thread-158 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicScanReader.workDone(AcidGuaranteesTestTool.java:302) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-24T22:14:08,717 DEBUG [Thread-158 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-24T22:14:08,717 DEBUG [Thread-158 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-24T22:14:08,717 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-24T22:14:08,864 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=40087 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=24 2024-11-24T22:14:08,864 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bd6592c37bb2:0-2 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732486426439.f0252adfc27b54a76a05585f3f45bf90. 2024-11-24T22:14:08,864 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bd6592c37bb2:0-2 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732486426439.f0252adfc27b54a76a05585f3f45bf90. as already flushing 2024-11-24T22:14:08,864 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bd6592c37bb2:0-2 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732486426439.f0252adfc27b54a76a05585f3f45bf90. 2024-11-24T22:14:08,865 ERROR [RS_FLUSH_OPERATIONS-regionserver/bd6592c37bb2:0-2 {event_type=RS_FLUSH_REGIONS, pid=24}] handler.RSProcedureHandler(58): pid=24 java.io.IOException: Unable to complete flush {ENCODED => f0252adfc27b54a76a05585f3f45bf90, NAME => 'TestAcidGuarantees,,1732486426439.f0252adfc27b54a76a05585f3f45bf90.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T22:14:08,865 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bd6592c37bb2:0-2 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=24 java.io.IOException: Unable to complete flush {ENCODED => f0252adfc27b54a76a05585f3f45bf90, NAME => 'TestAcidGuarantees,,1732486426439.f0252adfc27b54a76a05585f3f45bf90.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T22:14:08,865 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37701 {}] master.HMaster(4177): Remote procedure failed, pid=24 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => f0252adfc27b54a76a05585f3f45bf90, NAME => 'TestAcidGuarantees,,1732486426439.f0252adfc27b54a76a05585f3f45bf90.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => f0252adfc27b54a76a05585f3f45bf90, NAME => 'TestAcidGuarantees,,1732486426439.f0252adfc27b54a76a05585f3f45bf90.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T22:14:09,018 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=40087 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=24 2024-11-24T22:14:09,018 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bd6592c37bb2:0-0 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732486426439.f0252adfc27b54a76a05585f3f45bf90. 2024-11-24T22:14:09,018 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bd6592c37bb2:0-0 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732486426439.f0252adfc27b54a76a05585f3f45bf90. as already flushing 2024-11-24T22:14:09,018 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bd6592c37bb2:0-0 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732486426439.f0252adfc27b54a76a05585f3f45bf90. 2024-11-24T22:14:09,018 ERROR [RS_FLUSH_OPERATIONS-regionserver/bd6592c37bb2:0-0 {event_type=RS_FLUSH_REGIONS, pid=24}] handler.RSProcedureHandler(58): pid=24 java.io.IOException: Unable to complete flush {ENCODED => f0252adfc27b54a76a05585f3f45bf90, NAME => 'TestAcidGuarantees,,1732486426439.f0252adfc27b54a76a05585f3f45bf90.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T22:14:09,018 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bd6592c37bb2:0-0 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=24 java.io.IOException: Unable to complete flush {ENCODED => f0252adfc27b54a76a05585f3f45bf90, NAME => 'TestAcidGuarantees,,1732486426439.f0252adfc27b54a76a05585f3f45bf90.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T22:14:09,019 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37701 {}] master.HMaster(4177): Remote procedure failed, pid=24 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => f0252adfc27b54a76a05585f3f45bf90, NAME => 'TestAcidGuarantees,,1732486426439.f0252adfc27b54a76a05585f3f45bf90.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => f0252adfc27b54a76a05585f3f45bf90, NAME => 'TestAcidGuarantees,,1732486426439.f0252adfc27b54a76a05585f3f45bf90.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T22:14:09,071 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37701 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=23 2024-11-24T22:14:09,106 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=684 (bloomFilter=true), to=hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/f0252adfc27b54a76a05585f3f45bf90/.tmp/C/3fec91296d704444acf8427c12e1b565 2024-11-24T22:14:09,111 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/f0252adfc27b54a76a05585f3f45bf90/.tmp/A/2125dcf3fde74bf4bd202d0702d103aa as hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/f0252adfc27b54a76a05585f3f45bf90/A/2125dcf3fde74bf4bd202d0702d103aa 2024-11-24T22:14:09,116 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/f0252adfc27b54a76a05585f3f45bf90/A/2125dcf3fde74bf4bd202d0702d103aa, entries=150, sequenceid=684, filesize=12.0 K 2024-11-24T22:14:09,116 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/f0252adfc27b54a76a05585f3f45bf90/.tmp/B/6662a3106960477e8a8e7608436590b0 as hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/f0252adfc27b54a76a05585f3f45bf90/B/6662a3106960477e8a8e7608436590b0 2024-11-24T22:14:09,121 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/f0252adfc27b54a76a05585f3f45bf90/B/6662a3106960477e8a8e7608436590b0, entries=150, sequenceid=684, filesize=12.0 K 2024-11-24T22:14:09,121 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/f0252adfc27b54a76a05585f3f45bf90/.tmp/C/3fec91296d704444acf8427c12e1b565 as hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/f0252adfc27b54a76a05585f3f45bf90/C/3fec91296d704444acf8427c12e1b565 2024-11-24T22:14:09,125 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/f0252adfc27b54a76a05585f3f45bf90/C/3fec91296d704444acf8427c12e1b565, entries=150, sequenceid=684, filesize=12.0 K 2024-11-24T22:14:09,126 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~46.96 KB/48090, heapSize ~140.16 KB/143520, currentSize=134.18 KB/137400 for f0252adfc27b54a76a05585f3f45bf90 in 1310ms, sequenceid=684, compaction requested=true 2024-11-24T22:14:09,126 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for f0252adfc27b54a76a05585f3f45bf90: 2024-11-24T22:14:09,127 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store f0252adfc27b54a76a05585f3f45bf90:A, priority=-2147483648, current under compaction store size is 1 2024-11-24T22:14:09,127 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-24T22:14:09,127 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store f0252adfc27b54a76a05585f3f45bf90:B, priority=-2147483648, current under compaction store size is 2 2024-11-24T22:14:09,127 DEBUG [RS:0;bd6592c37bb2:40087-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-24T22:14:09,127 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-24T22:14:09,127 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store f0252adfc27b54a76a05585f3f45bf90:C, priority=-2147483648, current under compaction store size is 3 2024-11-24T22:14:09,127 DEBUG [RS:0;bd6592c37bb2:40087-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-24T22:14:09,127 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-24T22:14:09,128 DEBUG [RS:0;bd6592c37bb2:40087-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37823 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-24T22:14:09,128 DEBUG [RS:0;bd6592c37bb2:40087-longCompactions-0 {}] regionserver.HStore(1541): f0252adfc27b54a76a05585f3f45bf90/B is initiating minor compaction (all files) 2024-11-24T22:14:09,128 DEBUG [RS:0;bd6592c37bb2:40087-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37823 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-24T22:14:09,128 INFO [RS:0;bd6592c37bb2:40087-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of f0252adfc27b54a76a05585f3f45bf90/B in TestAcidGuarantees,,1732486426439.f0252adfc27b54a76a05585f3f45bf90. 2024-11-24T22:14:09,128 DEBUG [RS:0;bd6592c37bb2:40087-shortCompactions-0 {}] regionserver.HStore(1541): f0252adfc27b54a76a05585f3f45bf90/A is initiating minor compaction (all files) 2024-11-24T22:14:09,128 INFO [RS:0;bd6592c37bb2:40087-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of f0252adfc27b54a76a05585f3f45bf90/A in TestAcidGuarantees,,1732486426439.f0252adfc27b54a76a05585f3f45bf90. 2024-11-24T22:14:09,128 INFO [RS:0;bd6592c37bb2:40087-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/f0252adfc27b54a76a05585f3f45bf90/B/b390b104502d4d5393ffd3f302d7df00, hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/f0252adfc27b54a76a05585f3f45bf90/B/1496c056f69e420c9011e49538e1d877, hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/f0252adfc27b54a76a05585f3f45bf90/B/6662a3106960477e8a8e7608436590b0] into tmpdir=hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/f0252adfc27b54a76a05585f3f45bf90/.tmp, totalSize=36.9 K 2024-11-24T22:14:09,128 INFO [RS:0;bd6592c37bb2:40087-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/f0252adfc27b54a76a05585f3f45bf90/A/4b61696fbbe045e3b38c3abc9902e621, hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/f0252adfc27b54a76a05585f3f45bf90/A/de916a9109584ecba9ec30586fe8366b, hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/f0252adfc27b54a76a05585f3f45bf90/A/2125dcf3fde74bf4bd202d0702d103aa] into tmpdir=hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/f0252adfc27b54a76a05585f3f45bf90/.tmp, totalSize=36.9 K 2024-11-24T22:14:09,128 DEBUG [RS:0;bd6592c37bb2:40087-longCompactions-0 {}] compactions.Compactor(225): Compacting b390b104502d4d5393ffd3f302d7df00, keycount=150, bloomtype=ROW, size=12.9 K, encoding=NONE, compression=NONE, seqNum=637, earliestPutTs=1732486446547 2024-11-24T22:14:09,128 DEBUG [RS:0;bd6592c37bb2:40087-shortCompactions-0 {}] compactions.Compactor(225): Compacting 4b61696fbbe045e3b38c3abc9902e621, keycount=150, bloomtype=ROW, size=12.9 K, encoding=NONE, compression=NONE, seqNum=637, earliestPutTs=1732486446547 2024-11-24T22:14:09,129 DEBUG [RS:0;bd6592c37bb2:40087-longCompactions-0 {}] compactions.Compactor(225): Compacting 1496c056f69e420c9011e49538e1d877, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=671, earliestPutTs=1732486446645 2024-11-24T22:14:09,129 DEBUG [RS:0;bd6592c37bb2:40087-shortCompactions-0 {}] compactions.Compactor(225): Compacting de916a9109584ecba9ec30586fe8366b, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=671, earliestPutTs=1732486446645 2024-11-24T22:14:09,129 DEBUG [RS:0;bd6592c37bb2:40087-longCompactions-0 {}] compactions.Compactor(225): Compacting 6662a3106960477e8a8e7608436590b0, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=684, earliestPutTs=1732486447798 2024-11-24T22:14:09,129 DEBUG [RS:0;bd6592c37bb2:40087-shortCompactions-0 {}] compactions.Compactor(225): Compacting 2125dcf3fde74bf4bd202d0702d103aa, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=684, earliestPutTs=1732486447798 2024-11-24T22:14:09,132 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40087 {}] regionserver.HRegion(8855): Flush requested on f0252adfc27b54a76a05585f3f45bf90 2024-11-24T22:14:09,132 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing f0252adfc27b54a76a05585f3f45bf90 3/3 column families, dataSize=147.60 KB heapSize=415.45 KB 2024-11-24T22:14:09,133 INFO [Thread-145 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-145. 2024-11-24T22:14:09,133 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK f0252adfc27b54a76a05585f3f45bf90, store=A 2024-11-24T22:14:09,133 DEBUG [Thread-145 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-24T22:14:09,133 DEBUG [Thread-145 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-24T22:14:09,133 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-24T22:14:09,133 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK f0252adfc27b54a76a05585f3f45bf90, store=B 2024-11-24T22:14:09,133 DEBUG [Thread-145 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-24T22:14:09,133 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-24T22:14:09,133 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-24T22:14:09,133 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK f0252adfc27b54a76a05585f3f45bf90, store=C 2024-11-24T22:14:09,133 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-24T22:14:09,133 INFO [Thread-149 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-149. 2024-11-24T22:14:09,133 DEBUG [Thread-149 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-24T22:14:09,133 DEBUG [Thread-149 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-24T22:14:09,133 DEBUG [Thread-149 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-24T22:14:09,134 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-24T22:14:09,138 INFO [RS:0;bd6592c37bb2:40087-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): f0252adfc27b54a76a05585f3f45bf90#B#compaction#90 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-24T22:14:09,138 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/f0252adfc27b54a76a05585f3f45bf90/.tmp/A/c30a602de1784cbdba4e4748b7be0e24 is 50, key is test_row_0/A:col10/1732486447976/Put/seqid=0 2024-11-24T22:14:09,139 DEBUG [RS:0;bd6592c37bb2:40087-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/f0252adfc27b54a76a05585f3f45bf90/.tmp/B/37bf2eb16dda4726b76dda8535c368bf is 50, key is test_row_0/B:col10/1732486447802/Put/seqid=0 2024-11-24T22:14:09,141 INFO [RS:0;bd6592c37bb2:40087-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): f0252adfc27b54a76a05585f3f45bf90#A#compaction#92 average throughput is unlimited, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-24T22:14:09,141 DEBUG [RS:0;bd6592c37bb2:40087-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/f0252adfc27b54a76a05585f3f45bf90/.tmp/A/f140922ff681458cb1d780bb282b0389 is 50, key is test_row_0/A:col10/1732486447802/Put/seqid=0 2024-11-24T22:14:09,143 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45929 is added to blk_1073741927_1103 (size=12301) 2024-11-24T22:14:09,144 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45929 is added to blk_1073741928_1104 (size=13323) 2024-11-24T22:14:09,145 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=49.20 KB at sequenceid=721 (bloomFilter=true), to=hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/f0252adfc27b54a76a05585f3f45bf90/.tmp/A/c30a602de1784cbdba4e4748b7be0e24 2024-11-24T22:14:09,146 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45929 is added to blk_1073741929_1105 (size=13323) 2024-11-24T22:14:09,152 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/f0252adfc27b54a76a05585f3f45bf90/.tmp/B/fbe576cfb9064721b6aa3a1b5d3c25f2 is 50, key is test_row_0/B:col10/1732486447976/Put/seqid=0 2024-11-24T22:14:09,155 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45929 is added to blk_1073741930_1106 (size=12301) 2024-11-24T22:14:09,171 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=40087 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=24 2024-11-24T22:14:09,171 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bd6592c37bb2:0-1 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732486426439.f0252adfc27b54a76a05585f3f45bf90. 2024-11-24T22:14:09,171 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bd6592c37bb2:0-1 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732486426439.f0252adfc27b54a76a05585f3f45bf90. as already flushing 2024-11-24T22:14:09,172 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bd6592c37bb2:0-1 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732486426439.f0252adfc27b54a76a05585f3f45bf90. 2024-11-24T22:14:09,172 ERROR [RS_FLUSH_OPERATIONS-regionserver/bd6592c37bb2:0-1 {event_type=RS_FLUSH_REGIONS, pid=24}] handler.RSProcedureHandler(58): pid=24 java.io.IOException: Unable to complete flush {ENCODED => f0252adfc27b54a76a05585f3f45bf90, NAME => 'TestAcidGuarantees,,1732486426439.f0252adfc27b54a76a05585f3f45bf90.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T22:14:09,172 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bd6592c37bb2:0-1 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=24 java.io.IOException: Unable to complete flush {ENCODED => f0252adfc27b54a76a05585f3f45bf90, NAME => 'TestAcidGuarantees,,1732486426439.f0252adfc27b54a76a05585f3f45bf90.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T22:14:09,172 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37701 {}] master.HMaster(4177): Remote procedure failed, pid=24 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => f0252adfc27b54a76a05585f3f45bf90, NAME => 'TestAcidGuarantees,,1732486426439.f0252adfc27b54a76a05585f3f45bf90.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => f0252adfc27b54a76a05585f3f45bf90, NAME => 'TestAcidGuarantees,,1732486426439.f0252adfc27b54a76a05585f3f45bf90.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T22:14:09,325 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=40087 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=24 2024-11-24T22:14:09,325 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bd6592c37bb2:0-2 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732486426439.f0252adfc27b54a76a05585f3f45bf90. 2024-11-24T22:14:09,325 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bd6592c37bb2:0-2 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732486426439.f0252adfc27b54a76a05585f3f45bf90. as already flushing 2024-11-24T22:14:09,325 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bd6592c37bb2:0-2 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732486426439.f0252adfc27b54a76a05585f3f45bf90. 2024-11-24T22:14:09,325 ERROR [RS_FLUSH_OPERATIONS-regionserver/bd6592c37bb2:0-2 {event_type=RS_FLUSH_REGIONS, pid=24}] handler.RSProcedureHandler(58): pid=24 java.io.IOException: Unable to complete flush {ENCODED => f0252adfc27b54a76a05585f3f45bf90, NAME => 'TestAcidGuarantees,,1732486426439.f0252adfc27b54a76a05585f3f45bf90.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T22:14:09,325 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bd6592c37bb2:0-2 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=24 java.io.IOException: Unable to complete flush {ENCODED => f0252adfc27b54a76a05585f3f45bf90, NAME => 'TestAcidGuarantees,,1732486426439.f0252adfc27b54a76a05585f3f45bf90.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T22:14:09,326 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37701 {}] master.HMaster(4177): Remote procedure failed, pid=24 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => f0252adfc27b54a76a05585f3f45bf90, NAME => 'TestAcidGuarantees,,1732486426439.f0252adfc27b54a76a05585f3f45bf90.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => f0252adfc27b54a76a05585f3f45bf90, NAME => 'TestAcidGuarantees,,1732486426439.f0252adfc27b54a76a05585f3f45bf90.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T22:14:09,478 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=40087 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=24 2024-11-24T22:14:09,479 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bd6592c37bb2:0-0 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732486426439.f0252adfc27b54a76a05585f3f45bf90. 2024-11-24T22:14:09,479 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bd6592c37bb2:0-0 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732486426439.f0252adfc27b54a76a05585f3f45bf90. as already flushing 2024-11-24T22:14:09,479 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bd6592c37bb2:0-0 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732486426439.f0252adfc27b54a76a05585f3f45bf90. 2024-11-24T22:14:09,479 ERROR [RS_FLUSH_OPERATIONS-regionserver/bd6592c37bb2:0-0 {event_type=RS_FLUSH_REGIONS, pid=24}] handler.RSProcedureHandler(58): pid=24 java.io.IOException: Unable to complete flush {ENCODED => f0252adfc27b54a76a05585f3f45bf90, NAME => 'TestAcidGuarantees,,1732486426439.f0252adfc27b54a76a05585f3f45bf90.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T22:14:09,479 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bd6592c37bb2:0-0 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=24 java.io.IOException: Unable to complete flush {ENCODED => f0252adfc27b54a76a05585f3f45bf90, NAME => 'TestAcidGuarantees,,1732486426439.f0252adfc27b54a76a05585f3f45bf90.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T22:14:09,480 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37701 {}] master.HMaster(4177): Remote procedure failed, pid=24 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => f0252adfc27b54a76a05585f3f45bf90, NAME => 'TestAcidGuarantees,,1732486426439.f0252adfc27b54a76a05585f3f45bf90.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => f0252adfc27b54a76a05585f3f45bf90, NAME => 'TestAcidGuarantees,,1732486426439.f0252adfc27b54a76a05585f3f45bf90.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T22:14:09,550 DEBUG [RS:0;bd6592c37bb2:40087-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/f0252adfc27b54a76a05585f3f45bf90/.tmp/B/37bf2eb16dda4726b76dda8535c368bf as hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/f0252adfc27b54a76a05585f3f45bf90/B/37bf2eb16dda4726b76dda8535c368bf 2024-11-24T22:14:09,551 DEBUG [RS:0;bd6592c37bb2:40087-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/f0252adfc27b54a76a05585f3f45bf90/.tmp/A/f140922ff681458cb1d780bb282b0389 as hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/f0252adfc27b54a76a05585f3f45bf90/A/f140922ff681458cb1d780bb282b0389 2024-11-24T22:14:09,556 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=49.20 KB at sequenceid=721 (bloomFilter=true), to=hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/f0252adfc27b54a76a05585f3f45bf90/.tmp/B/fbe576cfb9064721b6aa3a1b5d3c25f2 2024-11-24T22:14:09,556 INFO [RS:0;bd6592c37bb2:40087-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in f0252adfc27b54a76a05585f3f45bf90/B of f0252adfc27b54a76a05585f3f45bf90 into 37bf2eb16dda4726b76dda8535c368bf(size=13.0 K), total size for store is 13.0 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-24T22:14:09,556 DEBUG [RS:0;bd6592c37bb2:40087-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for f0252adfc27b54a76a05585f3f45bf90: 2024-11-24T22:14:09,556 INFO [RS:0;bd6592c37bb2:40087-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732486426439.f0252adfc27b54a76a05585f3f45bf90., storeName=f0252adfc27b54a76a05585f3f45bf90/B, priority=13, startTime=1732486449127; duration=0sec 2024-11-24T22:14:09,557 DEBUG [RS:0;bd6592c37bb2:40087-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-24T22:14:09,557 DEBUG [RS:0;bd6592c37bb2:40087-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: f0252adfc27b54a76a05585f3f45bf90:B 2024-11-24T22:14:09,557 INFO [RS:0;bd6592c37bb2:40087-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in f0252adfc27b54a76a05585f3f45bf90/A of f0252adfc27b54a76a05585f3f45bf90 into f140922ff681458cb1d780bb282b0389(size=13.0 K), total size for store is 13.0 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-24T22:14:09,557 DEBUG [RS:0;bd6592c37bb2:40087-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-24T22:14:09,557 DEBUG [RS:0;bd6592c37bb2:40087-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for f0252adfc27b54a76a05585f3f45bf90: 2024-11-24T22:14:09,557 INFO [RS:0;bd6592c37bb2:40087-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732486426439.f0252adfc27b54a76a05585f3f45bf90., storeName=f0252adfc27b54a76a05585f3f45bf90/A, priority=13, startTime=1732486449127; duration=0sec 2024-11-24T22:14:09,557 DEBUG [RS:0;bd6592c37bb2:40087-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-24T22:14:09,557 DEBUG [RS:0;bd6592c37bb2:40087-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: f0252adfc27b54a76a05585f3f45bf90:A 2024-11-24T22:14:09,558 DEBUG [RS:0;bd6592c37bb2:40087-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37823 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-24T22:14:09,558 DEBUG [RS:0;bd6592c37bb2:40087-longCompactions-0 {}] regionserver.HStore(1541): f0252adfc27b54a76a05585f3f45bf90/C is initiating minor compaction (all files) 2024-11-24T22:14:09,558 INFO [RS:0;bd6592c37bb2:40087-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of f0252adfc27b54a76a05585f3f45bf90/C in TestAcidGuarantees,,1732486426439.f0252adfc27b54a76a05585f3f45bf90. 2024-11-24T22:14:09,558 INFO [RS:0;bd6592c37bb2:40087-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/f0252adfc27b54a76a05585f3f45bf90/C/830623b4e53f462483bc52b3a55ecc37, hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/f0252adfc27b54a76a05585f3f45bf90/C/94d5272cd82b4ec994e3c472dd9e27f2, hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/f0252adfc27b54a76a05585f3f45bf90/C/3fec91296d704444acf8427c12e1b565] into tmpdir=hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/f0252adfc27b54a76a05585f3f45bf90/.tmp, totalSize=36.9 K 2024-11-24T22:14:09,558 DEBUG [RS:0;bd6592c37bb2:40087-longCompactions-0 {}] compactions.Compactor(225): Compacting 830623b4e53f462483bc52b3a55ecc37, keycount=150, bloomtype=ROW, size=12.9 K, encoding=NONE, compression=NONE, seqNum=637, earliestPutTs=1732486446547 2024-11-24T22:14:09,559 DEBUG [RS:0;bd6592c37bb2:40087-longCompactions-0 {}] compactions.Compactor(225): Compacting 94d5272cd82b4ec994e3c472dd9e27f2, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=671, earliestPutTs=1732486446645 2024-11-24T22:14:09,559 DEBUG [RS:0;bd6592c37bb2:40087-longCompactions-0 {}] compactions.Compactor(225): Compacting 3fec91296d704444acf8427c12e1b565, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=684, earliestPutTs=1732486447798 2024-11-24T22:14:09,563 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/f0252adfc27b54a76a05585f3f45bf90/.tmp/C/02e6805bcf5747a79909959b2cfa0786 is 50, key is test_row_0/C:col10/1732486447976/Put/seqid=0 2024-11-24T22:14:09,566 INFO [RS:0;bd6592c37bb2:40087-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): f0252adfc27b54a76a05585f3f45bf90#C#compaction#95 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-24T22:14:09,566 DEBUG [RS:0;bd6592c37bb2:40087-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/f0252adfc27b54a76a05585f3f45bf90/.tmp/C/cc09dcf327b046d197b0e6a007e8c84d is 50, key is test_row_0/C:col10/1732486447802/Put/seqid=0 2024-11-24T22:14:09,567 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45929 is added to blk_1073741931_1107 (size=12301) 2024-11-24T22:14:09,569 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45929 is added to blk_1073741932_1108 (size=13323) 2024-11-24T22:14:09,632 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=40087 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=24 2024-11-24T22:14:09,632 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bd6592c37bb2:0-1 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732486426439.f0252adfc27b54a76a05585f3f45bf90. 2024-11-24T22:14:09,632 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bd6592c37bb2:0-1 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732486426439.f0252adfc27b54a76a05585f3f45bf90. as already flushing 2024-11-24T22:14:09,632 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bd6592c37bb2:0-1 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732486426439.f0252adfc27b54a76a05585f3f45bf90. 2024-11-24T22:14:09,632 ERROR [RS_FLUSH_OPERATIONS-regionserver/bd6592c37bb2:0-1 {event_type=RS_FLUSH_REGIONS, pid=24}] handler.RSProcedureHandler(58): pid=24 java.io.IOException: Unable to complete flush {ENCODED => f0252adfc27b54a76a05585f3f45bf90, NAME => 'TestAcidGuarantees,,1732486426439.f0252adfc27b54a76a05585f3f45bf90.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T22:14:09,633 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bd6592c37bb2:0-1 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=24 java.io.IOException: Unable to complete flush {ENCODED => f0252adfc27b54a76a05585f3f45bf90, NAME => 'TestAcidGuarantees,,1732486426439.f0252adfc27b54a76a05585f3f45bf90.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T22:14:09,633 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37701 {}] master.HMaster(4177): Remote procedure failed, pid=24 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => f0252adfc27b54a76a05585f3f45bf90, NAME => 'TestAcidGuarantees,,1732486426439.f0252adfc27b54a76a05585f3f45bf90.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => f0252adfc27b54a76a05585f3f45bf90, NAME => 'TestAcidGuarantees,,1732486426439.f0252adfc27b54a76a05585f3f45bf90.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T22:14:09,786 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=40087 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=24 2024-11-24T22:14:09,786 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bd6592c37bb2:0-2 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732486426439.f0252adfc27b54a76a05585f3f45bf90. 2024-11-24T22:14:09,786 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bd6592c37bb2:0-2 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732486426439.f0252adfc27b54a76a05585f3f45bf90. as already flushing 2024-11-24T22:14:09,786 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bd6592c37bb2:0-2 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732486426439.f0252adfc27b54a76a05585f3f45bf90. 2024-11-24T22:14:09,786 ERROR [RS_FLUSH_OPERATIONS-regionserver/bd6592c37bb2:0-2 {event_type=RS_FLUSH_REGIONS, pid=24}] handler.RSProcedureHandler(58): pid=24 java.io.IOException: Unable to complete flush {ENCODED => f0252adfc27b54a76a05585f3f45bf90, NAME => 'TestAcidGuarantees,,1732486426439.f0252adfc27b54a76a05585f3f45bf90.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T22:14:09,786 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bd6592c37bb2:0-2 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=24 java.io.IOException: Unable to complete flush {ENCODED => f0252adfc27b54a76a05585f3f45bf90, NAME => 'TestAcidGuarantees,,1732486426439.f0252adfc27b54a76a05585f3f45bf90.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T22:14:09,787 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37701 {}] master.HMaster(4177): Remote procedure failed, pid=24 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => f0252adfc27b54a76a05585f3f45bf90, NAME => 'TestAcidGuarantees,,1732486426439.f0252adfc27b54a76a05585f3f45bf90.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => f0252adfc27b54a76a05585f3f45bf90, NAME => 'TestAcidGuarantees,,1732486426439.f0252adfc27b54a76a05585f3f45bf90.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T22:14:09,957 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=40087 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=24 2024-11-24T22:14:09,957 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bd6592c37bb2:0-0 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732486426439.f0252adfc27b54a76a05585f3f45bf90. 2024-11-24T22:14:09,957 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bd6592c37bb2:0-0 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732486426439.f0252adfc27b54a76a05585f3f45bf90. as already flushing 2024-11-24T22:14:09,957 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bd6592c37bb2:0-0 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732486426439.f0252adfc27b54a76a05585f3f45bf90. 2024-11-24T22:14:09,958 ERROR [RS_FLUSH_OPERATIONS-regionserver/bd6592c37bb2:0-0 {event_type=RS_FLUSH_REGIONS, pid=24}] handler.RSProcedureHandler(58): pid=24 java.io.IOException: Unable to complete flush {ENCODED => f0252adfc27b54a76a05585f3f45bf90, NAME => 'TestAcidGuarantees,,1732486426439.f0252adfc27b54a76a05585f3f45bf90.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T22:14:09,958 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bd6592c37bb2:0-0 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=24 java.io.IOException: Unable to complete flush {ENCODED => f0252adfc27b54a76a05585f3f45bf90, NAME => 'TestAcidGuarantees,,1732486426439.f0252adfc27b54a76a05585f3f45bf90.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T22:14:09,959 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37701 {}] master.HMaster(4177): Remote procedure failed, pid=24 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => f0252adfc27b54a76a05585f3f45bf90, NAME => 'TestAcidGuarantees,,1732486426439.f0252adfc27b54a76a05585f3f45bf90.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => f0252adfc27b54a76a05585f3f45bf90, NAME => 'TestAcidGuarantees,,1732486426439.f0252adfc27b54a76a05585f3f45bf90.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T22:14:09,968 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=49.20 KB at sequenceid=721 (bloomFilter=true), to=hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/f0252adfc27b54a76a05585f3f45bf90/.tmp/C/02e6805bcf5747a79909959b2cfa0786 2024-11-24T22:14:09,974 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/f0252adfc27b54a76a05585f3f45bf90/.tmp/A/c30a602de1784cbdba4e4748b7be0e24 as hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/f0252adfc27b54a76a05585f3f45bf90/A/c30a602de1784cbdba4e4748b7be0e24 2024-11-24T22:14:09,975 DEBUG [RS:0;bd6592c37bb2:40087-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/f0252adfc27b54a76a05585f3f45bf90/.tmp/C/cc09dcf327b046d197b0e6a007e8c84d as hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/f0252adfc27b54a76a05585f3f45bf90/C/cc09dcf327b046d197b0e6a007e8c84d 2024-11-24T22:14:09,978 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/f0252adfc27b54a76a05585f3f45bf90/A/c30a602de1784cbdba4e4748b7be0e24, entries=150, sequenceid=721, filesize=12.0 K 2024-11-24T22:14:09,979 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/f0252adfc27b54a76a05585f3f45bf90/.tmp/B/fbe576cfb9064721b6aa3a1b5d3c25f2 as hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/f0252adfc27b54a76a05585f3f45bf90/B/fbe576cfb9064721b6aa3a1b5d3c25f2 2024-11-24T22:14:09,981 INFO [RS:0;bd6592c37bb2:40087-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in f0252adfc27b54a76a05585f3f45bf90/C of f0252adfc27b54a76a05585f3f45bf90 into cc09dcf327b046d197b0e6a007e8c84d(size=13.0 K), total size for store is 13.0 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-24T22:14:09,981 DEBUG [RS:0;bd6592c37bb2:40087-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for f0252adfc27b54a76a05585f3f45bf90: 2024-11-24T22:14:09,981 INFO [RS:0;bd6592c37bb2:40087-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732486426439.f0252adfc27b54a76a05585f3f45bf90., storeName=f0252adfc27b54a76a05585f3f45bf90/C, priority=13, startTime=1732486449127; duration=0sec 2024-11-24T22:14:09,981 DEBUG [RS:0;bd6592c37bb2:40087-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-24T22:14:09,981 DEBUG [RS:0;bd6592c37bb2:40087-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: f0252adfc27b54a76a05585f3f45bf90:C 2024-11-24T22:14:09,983 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/f0252adfc27b54a76a05585f3f45bf90/B/fbe576cfb9064721b6aa3a1b5d3c25f2, entries=150, sequenceid=721, filesize=12.0 K 2024-11-24T22:14:09,984 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/f0252adfc27b54a76a05585f3f45bf90/.tmp/C/02e6805bcf5747a79909959b2cfa0786 as hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/f0252adfc27b54a76a05585f3f45bf90/C/02e6805bcf5747a79909959b2cfa0786 2024-11-24T22:14:09,988 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/f0252adfc27b54a76a05585f3f45bf90/C/02e6805bcf5747a79909959b2cfa0786, entries=150, sequenceid=721, filesize=12.0 K 2024-11-24T22:14:09,989 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~147.60 KB/151140, heapSize ~415.41 KB/425376, currentSize=0 B/0 for f0252adfc27b54a76a05585f3f45bf90 in 857ms, sequenceid=721, compaction requested=false 2024-11-24T22:14:09,989 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for f0252adfc27b54a76a05585f3f45bf90: 2024-11-24T22:14:10,081 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37701 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=23 2024-11-24T22:14:10,111 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=40087 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=24 2024-11-24T22:14:10,111 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bd6592c37bb2:0-1 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732486426439.f0252adfc27b54a76a05585f3f45bf90. 2024-11-24T22:14:10,111 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bd6592c37bb2:0-1 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.HRegion(2603): Flush status journal for f0252adfc27b54a76a05585f3f45bf90: 2024-11-24T22:14:10,111 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bd6592c37bb2:0-1 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732486426439.f0252adfc27b54a76a05585f3f45bf90. 2024-11-24T22:14:10,111 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bd6592c37bb2:0-1 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=24 2024-11-24T22:14:10,112 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37701 {}] master.HMaster(4169): Remote procedure done, pid=24 2024-11-24T22:14:10,114 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=24, resume processing ppid=23 2024-11-24T22:14:10,114 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=24, ppid=23, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 2.1760 sec 2024-11-24T22:14:10,116 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=23, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=23, table=TestAcidGuarantees in 2.1830 sec 2024-11-24T22:14:12,015 DEBUG [FsDatasetAsyncDiskServiceFixer {}] hbase.HBaseTestingUtil$FsDatasetAsyncDiskServiceFixer(597): NoSuchFieldException: threadGroup; It might because your Hadoop version > 3.2.3 or 3.3.4, See HBASE-27595 for details. 2024-11-24T22:14:12,090 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37701 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=23 2024-11-24T22:14:12,091 INFO [RPCClient-NioEventLoopGroup-4-8 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-24T22:14:18,272 INFO [Thread-151 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-151. 2024-11-24T22:14:18,272 DEBUG [Thread-151 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-24T22:14:18,272 DEBUG [Thread-151 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-24T22:14:18,272 DEBUG [Thread-151 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-24T22:14:18,272 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-24T22:14:18,322 INFO [Thread-147 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-147. 2024-11-24T22:14:18,322 DEBUG [Thread-147 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-24T22:14:18,322 DEBUG [Thread-147 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-24T22:14:18,322 DEBUG [Thread-147 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-24T22:14:18,322 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-24T22:14:18,332 INFO [Thread-143 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-143. 2024-11-24T22:14:18,332 DEBUG [Thread-143 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-24T22:14:18,332 DEBUG [Thread-143 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-24T22:14:18,332 DEBUG [Thread-143 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-24T22:14:18,332 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-24T22:14:18,333 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(392): Finished test. Writers: 2024-11-24T22:14:18,333 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 63 2024-11-24T22:14:18,333 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 226 2024-11-24T22:14:18,333 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 80 2024-11-24T22:14:18,333 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 196 2024-11-24T22:14:18,333 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 65 2024-11-24T22:14:18,333 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(396): Readers: 2024-11-24T22:14:18,333 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(398): read 2335 2024-11-24T22:14:18,333 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(398): read 2361 2024-11-24T22:14:18,333 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(400): Scanners: 2024-11-24T22:14:18,333 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(402): scanned 1078 2024-11-24T22:14:18,333 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(403): verified 3230 rows 2024-11-24T22:14:18,333 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(402): scanned 1073 2024-11-24T22:14:18,333 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(403): verified 3212 rows 2024-11-24T22:14:18,333 INFO [Time-limited test {}] client.AsyncConnectionImpl(233): Connection has been closed by Time-limited test. 2024-11-24T22:14:18,333 DEBUG [Time-limited test {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool.doWork(AcidGuaranteesTestTool.java:131) at org.apache.hadoop.hbase.util.AbstractHBaseTool.run(AbstractHBaseTool.java:150) at org.apache.hadoop.hbase.AcidGuaranteesTestBase.runTestAtomicity(AcidGuaranteesTestBase.java:102) at org.apache.hadoop.hbase.AcidGuaranteesTestBase.runTestAtomicity(AcidGuaranteesTestBase.java:91) at org.apache.hadoop.hbase.AcidGuaranteesTestBase.testMixedAtomicity(AcidGuaranteesTestBase.java:117) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) at java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) at java.base/java.lang.reflect.Method.invoke(Method.java:568) at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) at java.base/java.util.concurrent.FutureTask.run(FutureTask.java:264) at java.base/java.lang.Thread.run(Thread.java:840) 2024-11-24T22:14:18,333 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-24T22:14:18,333 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-24T22:14:18,333 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-24T22:14:18,337 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37701 {}] master.HMaster$13(2820): Client=jenkins//172.17.0.2 disable TestAcidGuarantees 2024-11-24T22:14:18,342 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37701 {}] procedure2.ProcedureExecutor(1139): Stored pid=25, state=RUNNABLE:DISABLE_TABLE_PREPARE, hasLock=false; DisableTableProcedure table=TestAcidGuarantees 2024-11-24T22:14:18,345 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37701 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=25 2024-11-24T22:14:18,345 DEBUG [PEWorker-4 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1732486458345"}]},"ts":"1732486458345"} 2024-11-24T22:14:18,347 INFO [PEWorker-4 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=DISABLING in hbase:meta 2024-11-24T22:14:18,347 INFO [PEWorker-4 {}] procedure.DisableTableProcedure(284): Set TestAcidGuarantees to state=DISABLING 2024-11-24T22:14:18,348 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=26, ppid=25, state=RUNNABLE:CLOSE_TABLE_REGIONS_SCHEDULE, hasLock=false; CloseTableRegionsProcedure table=TestAcidGuarantees}] 2024-11-24T22:14:18,352 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=27, ppid=26, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=f0252adfc27b54a76a05585f3f45bf90, UNASSIGN}] 2024-11-24T22:14:18,353 INFO [PEWorker-2 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=27, ppid=26, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=f0252adfc27b54a76a05585f3f45bf90, UNASSIGN 2024-11-24T22:14:18,354 INFO [PEWorker-2 {}] assignment.RegionStateStore(223): pid=27 updating hbase:meta row=f0252adfc27b54a76a05585f3f45bf90, regionState=CLOSING, regionLocation=bd6592c37bb2,40087,1732486423713 2024-11-24T22:14:18,356 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=27, ppid=26, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=f0252adfc27b54a76a05585f3f45bf90, UNASSIGN because future has completed 2024-11-24T22:14:18,357 DEBUG [PEWorker-3 {}] assignment.TransitRegionStateProcedure(375): Close region: isSplit: false: evictOnSplit: true: evictOnClose: false 2024-11-24T22:14:18,357 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=28, ppid=27, state=RUNNABLE, hasLock=false; CloseRegionProcedure f0252adfc27b54a76a05585f3f45bf90, server=bd6592c37bb2,40087,1732486423713}] 2024-11-24T22:14:18,451 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37701 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=25 2024-11-24T22:14:18,513 INFO [RS_CLOSE_REGION-regionserver/bd6592c37bb2:0-0 {event_type=M_RS_CLOSE_REGION, pid=28}] handler.UnassignRegionHandler(122): Close f0252adfc27b54a76a05585f3f45bf90 2024-11-24T22:14:18,514 DEBUG [RS_CLOSE_REGION-regionserver/bd6592c37bb2:0-0 {event_type=M_RS_CLOSE_REGION, pid=28}] handler.UnassignRegionHandler(136): Unassign region: split region: false: evictCache: false 2024-11-24T22:14:18,514 DEBUG [RS_CLOSE_REGION-regionserver/bd6592c37bb2:0-0 {event_type=M_RS_CLOSE_REGION, pid=28}] regionserver.HRegion(1722): Closing f0252adfc27b54a76a05585f3f45bf90, disabling compactions & flushes 2024-11-24T22:14:18,514 INFO [RS_CLOSE_REGION-regionserver/bd6592c37bb2:0-0 {event_type=M_RS_CLOSE_REGION, pid=28}] regionserver.HRegion(1755): Closing region TestAcidGuarantees,,1732486426439.f0252adfc27b54a76a05585f3f45bf90. 2024-11-24T22:14:18,515 DEBUG [RS_CLOSE_REGION-regionserver/bd6592c37bb2:0-0 {event_type=M_RS_CLOSE_REGION, pid=28}] regionserver.HRegion(1776): Time limited wait for close lock on TestAcidGuarantees,,1732486426439.f0252adfc27b54a76a05585f3f45bf90. 2024-11-24T22:14:18,515 DEBUG [RS_CLOSE_REGION-regionserver/bd6592c37bb2:0-0 {event_type=M_RS_CLOSE_REGION, pid=28}] regionserver.HRegion(1843): Acquired close lock on TestAcidGuarantees,,1732486426439.f0252adfc27b54a76a05585f3f45bf90. after waiting 0 ms 2024-11-24T22:14:18,515 DEBUG [RS_CLOSE_REGION-regionserver/bd6592c37bb2:0-0 {event_type=M_RS_CLOSE_REGION, pid=28}] regionserver.HRegion(1853): Updates disabled for region TestAcidGuarantees,,1732486426439.f0252adfc27b54a76a05585f3f45bf90. 2024-11-24T22:14:18,515 INFO [RS_CLOSE_REGION-regionserver/bd6592c37bb2:0-0 {event_type=M_RS_CLOSE_REGION, pid=28}] regionserver.HRegion(2902): Flushing f0252adfc27b54a76a05585f3f45bf90 3/3 column families, dataSize=20.13 KB heapSize=60.52 KB 2024-11-24T22:14:18,515 DEBUG [RS_CLOSE_REGION-regionserver/bd6592c37bb2:0-0 {event_type=M_RS_CLOSE_REGION, pid=28}] regionserver.CompactingMemStore(205): FLUSHING TO DISK f0252adfc27b54a76a05585f3f45bf90, store=A 2024-11-24T22:14:18,515 DEBUG [RS_CLOSE_REGION-regionserver/bd6592c37bb2:0-0 {event_type=M_RS_CLOSE_REGION, pid=28}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-24T22:14:18,515 DEBUG [RS_CLOSE_REGION-regionserver/bd6592c37bb2:0-0 {event_type=M_RS_CLOSE_REGION, pid=28}] regionserver.CompactingMemStore(205): FLUSHING TO DISK f0252adfc27b54a76a05585f3f45bf90, store=B 2024-11-24T22:14:18,515 DEBUG [RS_CLOSE_REGION-regionserver/bd6592c37bb2:0-0 {event_type=M_RS_CLOSE_REGION, pid=28}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-24T22:14:18,515 DEBUG [RS_CLOSE_REGION-regionserver/bd6592c37bb2:0-0 {event_type=M_RS_CLOSE_REGION, pid=28}] regionserver.CompactingMemStore(205): FLUSHING TO DISK f0252adfc27b54a76a05585f3f45bf90, store=C 2024-11-24T22:14:18,515 DEBUG [RS_CLOSE_REGION-regionserver/bd6592c37bb2:0-0 {event_type=M_RS_CLOSE_REGION, pid=28}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-24T22:14:18,520 DEBUG [RS_CLOSE_REGION-regionserver/bd6592c37bb2:0-0 {event_type=M_RS_CLOSE_REGION, pid=28}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/f0252adfc27b54a76a05585f3f45bf90/.tmp/A/10d027912dea4ec1ad5bae02c41dea6e is 50, key is test_row_0/A:col10/1732486458331/Put/seqid=0 2024-11-24T22:14:18,524 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45929 is added to blk_1073741933_1109 (size=9857) 2024-11-24T22:14:18,661 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37701 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=25 2024-11-24T22:14:18,925 INFO [RS_CLOSE_REGION-regionserver/bd6592c37bb2:0-0 {event_type=M_RS_CLOSE_REGION, pid=28}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=6.71 KB at sequenceid=731 (bloomFilter=true), to=hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/f0252adfc27b54a76a05585f3f45bf90/.tmp/A/10d027912dea4ec1ad5bae02c41dea6e 2024-11-24T22:14:18,933 DEBUG [RS_CLOSE_REGION-regionserver/bd6592c37bb2:0-0 {event_type=M_RS_CLOSE_REGION, pid=28}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/f0252adfc27b54a76a05585f3f45bf90/.tmp/B/1abd35719ed74cc2a850acc71d60c279 is 50, key is test_row_0/B:col10/1732486458331/Put/seqid=0 2024-11-24T22:14:18,937 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45929 is added to blk_1073741934_1110 (size=9857) 2024-11-24T22:14:18,971 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37701 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=25 2024-11-24T22:14:19,338 INFO [RS_CLOSE_REGION-regionserver/bd6592c37bb2:0-0 {event_type=M_RS_CLOSE_REGION, pid=28}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=6.71 KB at sequenceid=731 (bloomFilter=true), to=hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/f0252adfc27b54a76a05585f3f45bf90/.tmp/B/1abd35719ed74cc2a850acc71d60c279 2024-11-24T22:14:19,346 DEBUG [RS_CLOSE_REGION-regionserver/bd6592c37bb2:0-0 {event_type=M_RS_CLOSE_REGION, pid=28}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/f0252adfc27b54a76a05585f3f45bf90/.tmp/C/61ab401cfaa3465fb546f51638a9b0ea is 50, key is test_row_0/C:col10/1732486458331/Put/seqid=0 2024-11-24T22:14:19,350 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45929 is added to blk_1073741935_1111 (size=9857) 2024-11-24T22:14:19,480 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37701 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=25 2024-11-24T22:14:19,750 INFO [RS_CLOSE_REGION-regionserver/bd6592c37bb2:0-0 {event_type=M_RS_CLOSE_REGION, pid=28}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=6.71 KB at sequenceid=731 (bloomFilter=true), to=hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/f0252adfc27b54a76a05585f3f45bf90/.tmp/C/61ab401cfaa3465fb546f51638a9b0ea 2024-11-24T22:14:19,756 DEBUG [RS_CLOSE_REGION-regionserver/bd6592c37bb2:0-0 {event_type=M_RS_CLOSE_REGION, pid=28}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/f0252adfc27b54a76a05585f3f45bf90/.tmp/A/10d027912dea4ec1ad5bae02c41dea6e as hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/f0252adfc27b54a76a05585f3f45bf90/A/10d027912dea4ec1ad5bae02c41dea6e 2024-11-24T22:14:19,760 INFO [RS_CLOSE_REGION-regionserver/bd6592c37bb2:0-0 {event_type=M_RS_CLOSE_REGION, pid=28}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/f0252adfc27b54a76a05585f3f45bf90/A/10d027912dea4ec1ad5bae02c41dea6e, entries=100, sequenceid=731, filesize=9.6 K 2024-11-24T22:14:19,761 DEBUG [RS_CLOSE_REGION-regionserver/bd6592c37bb2:0-0 {event_type=M_RS_CLOSE_REGION, pid=28}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/f0252adfc27b54a76a05585f3f45bf90/.tmp/B/1abd35719ed74cc2a850acc71d60c279 as hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/f0252adfc27b54a76a05585f3f45bf90/B/1abd35719ed74cc2a850acc71d60c279 2024-11-24T22:14:19,765 INFO [RS_CLOSE_REGION-regionserver/bd6592c37bb2:0-0 {event_type=M_RS_CLOSE_REGION, pid=28}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/f0252adfc27b54a76a05585f3f45bf90/B/1abd35719ed74cc2a850acc71d60c279, entries=100, sequenceid=731, filesize=9.6 K 2024-11-24T22:14:19,766 DEBUG [RS_CLOSE_REGION-regionserver/bd6592c37bb2:0-0 {event_type=M_RS_CLOSE_REGION, pid=28}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/f0252adfc27b54a76a05585f3f45bf90/.tmp/C/61ab401cfaa3465fb546f51638a9b0ea as hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/f0252adfc27b54a76a05585f3f45bf90/C/61ab401cfaa3465fb546f51638a9b0ea 2024-11-24T22:14:19,769 INFO [RS_CLOSE_REGION-regionserver/bd6592c37bb2:0-0 {event_type=M_RS_CLOSE_REGION, pid=28}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/f0252adfc27b54a76a05585f3f45bf90/C/61ab401cfaa3465fb546f51638a9b0ea, entries=100, sequenceid=731, filesize=9.6 K 2024-11-24T22:14:19,770 INFO [RS_CLOSE_REGION-regionserver/bd6592c37bb2:0-0 {event_type=M_RS_CLOSE_REGION, pid=28}] regionserver.HRegion(3140): Finished flush of dataSize ~20.13 KB/20610, heapSize ~60.47 KB/61920, currentSize=0 B/0 for f0252adfc27b54a76a05585f3f45bf90 in 1255ms, sequenceid=731, compaction requested=true 2024-11-24T22:14:19,771 DEBUG [StoreCloser-TestAcidGuarantees,,1732486426439.f0252adfc27b54a76a05585f3f45bf90.-1 {}] regionserver.HStore(2317): Moving the files [hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/f0252adfc27b54a76a05585f3f45bf90/A/85bc3afcd03143c1b1f0def65f0e2e63, hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/f0252adfc27b54a76a05585f3f45bf90/A/c148305567ff449293163aa4d9ca6086, hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/f0252adfc27b54a76a05585f3f45bf90/A/f86308fe4a9f4a84a5e46be6687402f0, hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/f0252adfc27b54a76a05585f3f45bf90/A/0a2765b1e92043a9b4d3fd87edcc95cd, hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/f0252adfc27b54a76a05585f3f45bf90/A/705ad699596346dba6cf6781c5011618, hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/f0252adfc27b54a76a05585f3f45bf90/A/c8cf77fc903c408d900988a3a2b6a93c, hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/f0252adfc27b54a76a05585f3f45bf90/A/ab0287723e304b5ba215650833ca6579, hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/f0252adfc27b54a76a05585f3f45bf90/A/b146b26e6c524e5ca691fb2961fc5ffc, hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/f0252adfc27b54a76a05585f3f45bf90/A/f3da0b40cc124272a32346cc906d98c5, hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/f0252adfc27b54a76a05585f3f45bf90/A/36b23a52fa034f5c9277bf34526f85c8, hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/f0252adfc27b54a76a05585f3f45bf90/A/999745bc0c3c48e495f7184a9cd014a8, hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/f0252adfc27b54a76a05585f3f45bf90/A/8758c2800adf4f4dba091cf4fb29d2f3, hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/f0252adfc27b54a76a05585f3f45bf90/A/c2bd8d33a4064e3b8a2f0df67e949fa7, hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/f0252adfc27b54a76a05585f3f45bf90/A/7f4a2c729d3b4d5bb3dacfc98aacdcfc, hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/f0252adfc27b54a76a05585f3f45bf90/A/858d340a75944cf79e54a6945720ed2e, hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/f0252adfc27b54a76a05585f3f45bf90/A/5bcdeaa73e0f49c29d59c30322b2c997, hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/f0252adfc27b54a76a05585f3f45bf90/A/7160c90e6125430f851ec64126d011ae, hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/f0252adfc27b54a76a05585f3f45bf90/A/1dc58facd3964ecdb9a0f4fffee4cfe6, hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/f0252adfc27b54a76a05585f3f45bf90/A/f4702fb2e5c54e189b3430b90cd93ef8, hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/f0252adfc27b54a76a05585f3f45bf90/A/7e8bd1915fc94a819d890f3dcfe52f79, hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/f0252adfc27b54a76a05585f3f45bf90/A/c6b8d474fb464a6dbbb00cd9d0d18449, hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/f0252adfc27b54a76a05585f3f45bf90/A/749bc91e4eac4c2894066ef9bc5b64b6, hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/f0252adfc27b54a76a05585f3f45bf90/A/1a617bcb02d94acf9f84238fe3511506, hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/f0252adfc27b54a76a05585f3f45bf90/A/da0dc8e8af434e93b77478eb2c1ab439, hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/f0252adfc27b54a76a05585f3f45bf90/A/bb4373777b7749fb9ffb570fede7c5e7, hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/f0252adfc27b54a76a05585f3f45bf90/A/cfea60b26a5e4f2694b58c5df1e8f3d2, hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/f0252adfc27b54a76a05585f3f45bf90/A/4b61696fbbe045e3b38c3abc9902e621, hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/f0252adfc27b54a76a05585f3f45bf90/A/efb9cfb73d5e4768bbda183d1331835c, hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/f0252adfc27b54a76a05585f3f45bf90/A/de916a9109584ecba9ec30586fe8366b, hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/f0252adfc27b54a76a05585f3f45bf90/A/2125dcf3fde74bf4bd202d0702d103aa] to archive 2024-11-24T22:14:19,774 DEBUG [StoreCloser-TestAcidGuarantees,,1732486426439.f0252adfc27b54a76a05585f3f45bf90.-1 {}] backup.HFileArchiver(360): Archiving compacted files. 2024-11-24T22:14:19,779 DEBUG [StoreCloser-TestAcidGuarantees,,1732486426439.f0252adfc27b54a76a05585f3f45bf90.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/f0252adfc27b54a76a05585f3f45bf90/A/85bc3afcd03143c1b1f0def65f0e2e63 to hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/archive/data/default/TestAcidGuarantees/f0252adfc27b54a76a05585f3f45bf90/A/85bc3afcd03143c1b1f0def65f0e2e63 2024-11-24T22:14:19,780 DEBUG [StoreCloser-TestAcidGuarantees,,1732486426439.f0252adfc27b54a76a05585f3f45bf90.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/f0252adfc27b54a76a05585f3f45bf90/A/c148305567ff449293163aa4d9ca6086 to hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/archive/data/default/TestAcidGuarantees/f0252adfc27b54a76a05585f3f45bf90/A/c148305567ff449293163aa4d9ca6086 2024-11-24T22:14:19,782 DEBUG [StoreCloser-TestAcidGuarantees,,1732486426439.f0252adfc27b54a76a05585f3f45bf90.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/f0252adfc27b54a76a05585f3f45bf90/A/f86308fe4a9f4a84a5e46be6687402f0 to hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/archive/data/default/TestAcidGuarantees/f0252adfc27b54a76a05585f3f45bf90/A/f86308fe4a9f4a84a5e46be6687402f0 2024-11-24T22:14:19,783 DEBUG [StoreCloser-TestAcidGuarantees,,1732486426439.f0252adfc27b54a76a05585f3f45bf90.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/f0252adfc27b54a76a05585f3f45bf90/A/0a2765b1e92043a9b4d3fd87edcc95cd to hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/archive/data/default/TestAcidGuarantees/f0252adfc27b54a76a05585f3f45bf90/A/0a2765b1e92043a9b4d3fd87edcc95cd 2024-11-24T22:14:19,784 DEBUG [StoreCloser-TestAcidGuarantees,,1732486426439.f0252adfc27b54a76a05585f3f45bf90.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/f0252adfc27b54a76a05585f3f45bf90/A/705ad699596346dba6cf6781c5011618 to hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/archive/data/default/TestAcidGuarantees/f0252adfc27b54a76a05585f3f45bf90/A/705ad699596346dba6cf6781c5011618 2024-11-24T22:14:19,785 DEBUG [StoreCloser-TestAcidGuarantees,,1732486426439.f0252adfc27b54a76a05585f3f45bf90.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/f0252adfc27b54a76a05585f3f45bf90/A/c8cf77fc903c408d900988a3a2b6a93c to hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/archive/data/default/TestAcidGuarantees/f0252adfc27b54a76a05585f3f45bf90/A/c8cf77fc903c408d900988a3a2b6a93c 2024-11-24T22:14:19,786 DEBUG [StoreCloser-TestAcidGuarantees,,1732486426439.f0252adfc27b54a76a05585f3f45bf90.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/f0252adfc27b54a76a05585f3f45bf90/A/ab0287723e304b5ba215650833ca6579 to hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/archive/data/default/TestAcidGuarantees/f0252adfc27b54a76a05585f3f45bf90/A/ab0287723e304b5ba215650833ca6579 2024-11-24T22:14:19,788 DEBUG [StoreCloser-TestAcidGuarantees,,1732486426439.f0252adfc27b54a76a05585f3f45bf90.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/f0252adfc27b54a76a05585f3f45bf90/A/b146b26e6c524e5ca691fb2961fc5ffc to hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/archive/data/default/TestAcidGuarantees/f0252adfc27b54a76a05585f3f45bf90/A/b146b26e6c524e5ca691fb2961fc5ffc 2024-11-24T22:14:19,789 DEBUG [StoreCloser-TestAcidGuarantees,,1732486426439.f0252adfc27b54a76a05585f3f45bf90.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/f0252adfc27b54a76a05585f3f45bf90/A/f3da0b40cc124272a32346cc906d98c5 to hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/archive/data/default/TestAcidGuarantees/f0252adfc27b54a76a05585f3f45bf90/A/f3da0b40cc124272a32346cc906d98c5 2024-11-24T22:14:19,790 DEBUG [StoreCloser-TestAcidGuarantees,,1732486426439.f0252adfc27b54a76a05585f3f45bf90.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/f0252adfc27b54a76a05585f3f45bf90/A/36b23a52fa034f5c9277bf34526f85c8 to hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/archive/data/default/TestAcidGuarantees/f0252adfc27b54a76a05585f3f45bf90/A/36b23a52fa034f5c9277bf34526f85c8 2024-11-24T22:14:19,791 DEBUG [StoreCloser-TestAcidGuarantees,,1732486426439.f0252adfc27b54a76a05585f3f45bf90.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/f0252adfc27b54a76a05585f3f45bf90/A/999745bc0c3c48e495f7184a9cd014a8 to hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/archive/data/default/TestAcidGuarantees/f0252adfc27b54a76a05585f3f45bf90/A/999745bc0c3c48e495f7184a9cd014a8 2024-11-24T22:14:19,792 DEBUG [StoreCloser-TestAcidGuarantees,,1732486426439.f0252adfc27b54a76a05585f3f45bf90.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/f0252adfc27b54a76a05585f3f45bf90/A/8758c2800adf4f4dba091cf4fb29d2f3 to hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/archive/data/default/TestAcidGuarantees/f0252adfc27b54a76a05585f3f45bf90/A/8758c2800adf4f4dba091cf4fb29d2f3 2024-11-24T22:14:19,793 DEBUG [StoreCloser-TestAcidGuarantees,,1732486426439.f0252adfc27b54a76a05585f3f45bf90.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/f0252adfc27b54a76a05585f3f45bf90/A/c2bd8d33a4064e3b8a2f0df67e949fa7 to hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/archive/data/default/TestAcidGuarantees/f0252adfc27b54a76a05585f3f45bf90/A/c2bd8d33a4064e3b8a2f0df67e949fa7 2024-11-24T22:14:19,794 DEBUG [StoreCloser-TestAcidGuarantees,,1732486426439.f0252adfc27b54a76a05585f3f45bf90.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/f0252adfc27b54a76a05585f3f45bf90/A/7f4a2c729d3b4d5bb3dacfc98aacdcfc to hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/archive/data/default/TestAcidGuarantees/f0252adfc27b54a76a05585f3f45bf90/A/7f4a2c729d3b4d5bb3dacfc98aacdcfc 2024-11-24T22:14:19,795 DEBUG [StoreCloser-TestAcidGuarantees,,1732486426439.f0252adfc27b54a76a05585f3f45bf90.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/f0252adfc27b54a76a05585f3f45bf90/A/858d340a75944cf79e54a6945720ed2e to hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/archive/data/default/TestAcidGuarantees/f0252adfc27b54a76a05585f3f45bf90/A/858d340a75944cf79e54a6945720ed2e 2024-11-24T22:14:19,796 DEBUG [StoreCloser-TestAcidGuarantees,,1732486426439.f0252adfc27b54a76a05585f3f45bf90.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/f0252adfc27b54a76a05585f3f45bf90/A/5bcdeaa73e0f49c29d59c30322b2c997 to hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/archive/data/default/TestAcidGuarantees/f0252adfc27b54a76a05585f3f45bf90/A/5bcdeaa73e0f49c29d59c30322b2c997 2024-11-24T22:14:19,798 DEBUG [StoreCloser-TestAcidGuarantees,,1732486426439.f0252adfc27b54a76a05585f3f45bf90.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/f0252adfc27b54a76a05585f3f45bf90/A/7160c90e6125430f851ec64126d011ae to hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/archive/data/default/TestAcidGuarantees/f0252adfc27b54a76a05585f3f45bf90/A/7160c90e6125430f851ec64126d011ae 2024-11-24T22:14:19,799 DEBUG [StoreCloser-TestAcidGuarantees,,1732486426439.f0252adfc27b54a76a05585f3f45bf90.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/f0252adfc27b54a76a05585f3f45bf90/A/1dc58facd3964ecdb9a0f4fffee4cfe6 to hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/archive/data/default/TestAcidGuarantees/f0252adfc27b54a76a05585f3f45bf90/A/1dc58facd3964ecdb9a0f4fffee4cfe6 2024-11-24T22:14:19,800 DEBUG [StoreCloser-TestAcidGuarantees,,1732486426439.f0252adfc27b54a76a05585f3f45bf90.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/f0252adfc27b54a76a05585f3f45bf90/A/f4702fb2e5c54e189b3430b90cd93ef8 to hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/archive/data/default/TestAcidGuarantees/f0252adfc27b54a76a05585f3f45bf90/A/f4702fb2e5c54e189b3430b90cd93ef8 2024-11-24T22:14:19,801 DEBUG [StoreCloser-TestAcidGuarantees,,1732486426439.f0252adfc27b54a76a05585f3f45bf90.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/f0252adfc27b54a76a05585f3f45bf90/A/7e8bd1915fc94a819d890f3dcfe52f79 to hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/archive/data/default/TestAcidGuarantees/f0252adfc27b54a76a05585f3f45bf90/A/7e8bd1915fc94a819d890f3dcfe52f79 2024-11-24T22:14:19,802 DEBUG [StoreCloser-TestAcidGuarantees,,1732486426439.f0252adfc27b54a76a05585f3f45bf90.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/f0252adfc27b54a76a05585f3f45bf90/A/c6b8d474fb464a6dbbb00cd9d0d18449 to hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/archive/data/default/TestAcidGuarantees/f0252adfc27b54a76a05585f3f45bf90/A/c6b8d474fb464a6dbbb00cd9d0d18449 2024-11-24T22:14:19,803 DEBUG [StoreCloser-TestAcidGuarantees,,1732486426439.f0252adfc27b54a76a05585f3f45bf90.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/f0252adfc27b54a76a05585f3f45bf90/A/749bc91e4eac4c2894066ef9bc5b64b6 to hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/archive/data/default/TestAcidGuarantees/f0252adfc27b54a76a05585f3f45bf90/A/749bc91e4eac4c2894066ef9bc5b64b6 2024-11-24T22:14:19,804 DEBUG [StoreCloser-TestAcidGuarantees,,1732486426439.f0252adfc27b54a76a05585f3f45bf90.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/f0252adfc27b54a76a05585f3f45bf90/A/1a617bcb02d94acf9f84238fe3511506 to hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/archive/data/default/TestAcidGuarantees/f0252adfc27b54a76a05585f3f45bf90/A/1a617bcb02d94acf9f84238fe3511506 2024-11-24T22:14:19,805 DEBUG [StoreCloser-TestAcidGuarantees,,1732486426439.f0252adfc27b54a76a05585f3f45bf90.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/f0252adfc27b54a76a05585f3f45bf90/A/da0dc8e8af434e93b77478eb2c1ab439 to hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/archive/data/default/TestAcidGuarantees/f0252adfc27b54a76a05585f3f45bf90/A/da0dc8e8af434e93b77478eb2c1ab439 2024-11-24T22:14:19,806 DEBUG [StoreCloser-TestAcidGuarantees,,1732486426439.f0252adfc27b54a76a05585f3f45bf90.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/f0252adfc27b54a76a05585f3f45bf90/A/bb4373777b7749fb9ffb570fede7c5e7 to hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/archive/data/default/TestAcidGuarantees/f0252adfc27b54a76a05585f3f45bf90/A/bb4373777b7749fb9ffb570fede7c5e7 2024-11-24T22:14:19,807 DEBUG [StoreCloser-TestAcidGuarantees,,1732486426439.f0252adfc27b54a76a05585f3f45bf90.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/f0252adfc27b54a76a05585f3f45bf90/A/cfea60b26a5e4f2694b58c5df1e8f3d2 to hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/archive/data/default/TestAcidGuarantees/f0252adfc27b54a76a05585f3f45bf90/A/cfea60b26a5e4f2694b58c5df1e8f3d2 2024-11-24T22:14:19,809 DEBUG [StoreCloser-TestAcidGuarantees,,1732486426439.f0252adfc27b54a76a05585f3f45bf90.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/f0252adfc27b54a76a05585f3f45bf90/A/4b61696fbbe045e3b38c3abc9902e621 to hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/archive/data/default/TestAcidGuarantees/f0252adfc27b54a76a05585f3f45bf90/A/4b61696fbbe045e3b38c3abc9902e621 2024-11-24T22:14:19,810 DEBUG [StoreCloser-TestAcidGuarantees,,1732486426439.f0252adfc27b54a76a05585f3f45bf90.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/f0252adfc27b54a76a05585f3f45bf90/A/efb9cfb73d5e4768bbda183d1331835c to hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/archive/data/default/TestAcidGuarantees/f0252adfc27b54a76a05585f3f45bf90/A/efb9cfb73d5e4768bbda183d1331835c 2024-11-24T22:14:19,811 DEBUG [StoreCloser-TestAcidGuarantees,,1732486426439.f0252adfc27b54a76a05585f3f45bf90.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/f0252adfc27b54a76a05585f3f45bf90/A/de916a9109584ecba9ec30586fe8366b to hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/archive/data/default/TestAcidGuarantees/f0252adfc27b54a76a05585f3f45bf90/A/de916a9109584ecba9ec30586fe8366b 2024-11-24T22:14:19,812 DEBUG [StoreCloser-TestAcidGuarantees,,1732486426439.f0252adfc27b54a76a05585f3f45bf90.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/f0252adfc27b54a76a05585f3f45bf90/A/2125dcf3fde74bf4bd202d0702d103aa to hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/archive/data/default/TestAcidGuarantees/f0252adfc27b54a76a05585f3f45bf90/A/2125dcf3fde74bf4bd202d0702d103aa 2024-11-24T22:14:19,828 DEBUG [StoreCloser-TestAcidGuarantees,,1732486426439.f0252adfc27b54a76a05585f3f45bf90.-1 {}] regionserver.HStore(2317): Moving the files [hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/f0252adfc27b54a76a05585f3f45bf90/B/fa858762490548298016038bb1fe3609, hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/f0252adfc27b54a76a05585f3f45bf90/B/b7e5d8cf64e84135afbc9fb3af466717, hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/f0252adfc27b54a76a05585f3f45bf90/B/6cdf2f3301bb413f9d1e2d9b0160fee3, hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/f0252adfc27b54a76a05585f3f45bf90/B/5609bc3ed6cb49db83278dc1722a8dbb, hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/f0252adfc27b54a76a05585f3f45bf90/B/f77f1f3e41f74085b82344614b253a1f, hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/f0252adfc27b54a76a05585f3f45bf90/B/be5ac8db548f4f02af9d4afa54364feb, hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/f0252adfc27b54a76a05585f3f45bf90/B/dc390cb315b24517a00c5d625773456e, hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/f0252adfc27b54a76a05585f3f45bf90/B/0f8a8c8188ae448487505488e234be20, hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/f0252adfc27b54a76a05585f3f45bf90/B/ffc9abede25d4d48a0ffc2479587f52a, hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/f0252adfc27b54a76a05585f3f45bf90/B/f2c3972b4102489780168f825b2d9169, hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/f0252adfc27b54a76a05585f3f45bf90/B/7fd5a8a6e6934156aebd2bd7ed3c6562, hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/f0252adfc27b54a76a05585f3f45bf90/B/2e8f14af0ec94e6bb8136e2bdd679ca3, hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/f0252adfc27b54a76a05585f3f45bf90/B/9e9ff2c611584faf92b0cdcb1129728e, hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/f0252adfc27b54a76a05585f3f45bf90/B/3710459809154d628174fac723311f35, hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/f0252adfc27b54a76a05585f3f45bf90/B/a57df684ebd4407e9ca9d68f74a40615, hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/f0252adfc27b54a76a05585f3f45bf90/B/56630cf0374945f290b0ada498ece426, hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/f0252adfc27b54a76a05585f3f45bf90/B/f786dc1420844a9da33ca12b1b422ea3, hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/f0252adfc27b54a76a05585f3f45bf90/B/417bf79f9dda4788b24c3865cecc3237, hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/f0252adfc27b54a76a05585f3f45bf90/B/7ddf65b17c1948cb8dac4dce2ff012c6, hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/f0252adfc27b54a76a05585f3f45bf90/B/da93b0c6087f4675a21ee70eb6f87149, hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/f0252adfc27b54a76a05585f3f45bf90/B/70abb3621bf148439ad48345232947f0, hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/f0252adfc27b54a76a05585f3f45bf90/B/e63699926e974340b82e6c44beff995b, hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/f0252adfc27b54a76a05585f3f45bf90/B/cc7c51615aeb4494be1ea60782c5134a, hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/f0252adfc27b54a76a05585f3f45bf90/B/4d4d753c5b6141dbba6ad07b969acd4b, hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/f0252adfc27b54a76a05585f3f45bf90/B/13d49dd09c36477fa2ccd8d4b6050657, hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/f0252adfc27b54a76a05585f3f45bf90/B/60727be57d64464b947b404585a28958, hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/f0252adfc27b54a76a05585f3f45bf90/B/b390b104502d4d5393ffd3f302d7df00, hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/f0252adfc27b54a76a05585f3f45bf90/B/81b92b86e15441e090171e98694d5519, hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/f0252adfc27b54a76a05585f3f45bf90/B/1496c056f69e420c9011e49538e1d877, hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/f0252adfc27b54a76a05585f3f45bf90/B/6662a3106960477e8a8e7608436590b0] to archive 2024-11-24T22:14:19,829 DEBUG [StoreCloser-TestAcidGuarantees,,1732486426439.f0252adfc27b54a76a05585f3f45bf90.-1 {}] backup.HFileArchiver(360): Archiving compacted files. 2024-11-24T22:14:19,831 DEBUG [StoreCloser-TestAcidGuarantees,,1732486426439.f0252adfc27b54a76a05585f3f45bf90.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/f0252adfc27b54a76a05585f3f45bf90/B/fa858762490548298016038bb1fe3609 to hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/archive/data/default/TestAcidGuarantees/f0252adfc27b54a76a05585f3f45bf90/B/fa858762490548298016038bb1fe3609 2024-11-24T22:14:19,832 DEBUG [StoreCloser-TestAcidGuarantees,,1732486426439.f0252adfc27b54a76a05585f3f45bf90.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/f0252adfc27b54a76a05585f3f45bf90/B/b7e5d8cf64e84135afbc9fb3af466717 to hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/archive/data/default/TestAcidGuarantees/f0252adfc27b54a76a05585f3f45bf90/B/b7e5d8cf64e84135afbc9fb3af466717 2024-11-24T22:14:19,834 DEBUG [StoreCloser-TestAcidGuarantees,,1732486426439.f0252adfc27b54a76a05585f3f45bf90.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/f0252adfc27b54a76a05585f3f45bf90/B/6cdf2f3301bb413f9d1e2d9b0160fee3 to hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/archive/data/default/TestAcidGuarantees/f0252adfc27b54a76a05585f3f45bf90/B/6cdf2f3301bb413f9d1e2d9b0160fee3 2024-11-24T22:14:19,835 DEBUG [StoreCloser-TestAcidGuarantees,,1732486426439.f0252adfc27b54a76a05585f3f45bf90.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/f0252adfc27b54a76a05585f3f45bf90/B/5609bc3ed6cb49db83278dc1722a8dbb to hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/archive/data/default/TestAcidGuarantees/f0252adfc27b54a76a05585f3f45bf90/B/5609bc3ed6cb49db83278dc1722a8dbb 2024-11-24T22:14:19,836 DEBUG [StoreCloser-TestAcidGuarantees,,1732486426439.f0252adfc27b54a76a05585f3f45bf90.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/f0252adfc27b54a76a05585f3f45bf90/B/f77f1f3e41f74085b82344614b253a1f to hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/archive/data/default/TestAcidGuarantees/f0252adfc27b54a76a05585f3f45bf90/B/f77f1f3e41f74085b82344614b253a1f 2024-11-24T22:14:19,838 DEBUG [StoreCloser-TestAcidGuarantees,,1732486426439.f0252adfc27b54a76a05585f3f45bf90.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/f0252adfc27b54a76a05585f3f45bf90/B/be5ac8db548f4f02af9d4afa54364feb to hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/archive/data/default/TestAcidGuarantees/f0252adfc27b54a76a05585f3f45bf90/B/be5ac8db548f4f02af9d4afa54364feb 2024-11-24T22:14:19,839 DEBUG [StoreCloser-TestAcidGuarantees,,1732486426439.f0252adfc27b54a76a05585f3f45bf90.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/f0252adfc27b54a76a05585f3f45bf90/B/dc390cb315b24517a00c5d625773456e to hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/archive/data/default/TestAcidGuarantees/f0252adfc27b54a76a05585f3f45bf90/B/dc390cb315b24517a00c5d625773456e 2024-11-24T22:14:19,841 DEBUG [StoreCloser-TestAcidGuarantees,,1732486426439.f0252adfc27b54a76a05585f3f45bf90.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/f0252adfc27b54a76a05585f3f45bf90/B/0f8a8c8188ae448487505488e234be20 to hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/archive/data/default/TestAcidGuarantees/f0252adfc27b54a76a05585f3f45bf90/B/0f8a8c8188ae448487505488e234be20 2024-11-24T22:14:19,842 DEBUG [StoreCloser-TestAcidGuarantees,,1732486426439.f0252adfc27b54a76a05585f3f45bf90.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/f0252adfc27b54a76a05585f3f45bf90/B/ffc9abede25d4d48a0ffc2479587f52a to hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/archive/data/default/TestAcidGuarantees/f0252adfc27b54a76a05585f3f45bf90/B/ffc9abede25d4d48a0ffc2479587f52a 2024-11-24T22:14:19,843 DEBUG [StoreCloser-TestAcidGuarantees,,1732486426439.f0252adfc27b54a76a05585f3f45bf90.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/f0252adfc27b54a76a05585f3f45bf90/B/f2c3972b4102489780168f825b2d9169 to hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/archive/data/default/TestAcidGuarantees/f0252adfc27b54a76a05585f3f45bf90/B/f2c3972b4102489780168f825b2d9169 2024-11-24T22:14:19,844 DEBUG [StoreCloser-TestAcidGuarantees,,1732486426439.f0252adfc27b54a76a05585f3f45bf90.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/f0252adfc27b54a76a05585f3f45bf90/B/7fd5a8a6e6934156aebd2bd7ed3c6562 to hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/archive/data/default/TestAcidGuarantees/f0252adfc27b54a76a05585f3f45bf90/B/7fd5a8a6e6934156aebd2bd7ed3c6562 2024-11-24T22:14:19,846 DEBUG [StoreCloser-TestAcidGuarantees,,1732486426439.f0252adfc27b54a76a05585f3f45bf90.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/f0252adfc27b54a76a05585f3f45bf90/B/2e8f14af0ec94e6bb8136e2bdd679ca3 to hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/archive/data/default/TestAcidGuarantees/f0252adfc27b54a76a05585f3f45bf90/B/2e8f14af0ec94e6bb8136e2bdd679ca3 2024-11-24T22:14:19,847 DEBUG [StoreCloser-TestAcidGuarantees,,1732486426439.f0252adfc27b54a76a05585f3f45bf90.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/f0252adfc27b54a76a05585f3f45bf90/B/9e9ff2c611584faf92b0cdcb1129728e to hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/archive/data/default/TestAcidGuarantees/f0252adfc27b54a76a05585f3f45bf90/B/9e9ff2c611584faf92b0cdcb1129728e 2024-11-24T22:14:19,848 DEBUG [StoreCloser-TestAcidGuarantees,,1732486426439.f0252adfc27b54a76a05585f3f45bf90.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/f0252adfc27b54a76a05585f3f45bf90/B/3710459809154d628174fac723311f35 to hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/archive/data/default/TestAcidGuarantees/f0252adfc27b54a76a05585f3f45bf90/B/3710459809154d628174fac723311f35 2024-11-24T22:14:19,850 DEBUG [StoreCloser-TestAcidGuarantees,,1732486426439.f0252adfc27b54a76a05585f3f45bf90.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/f0252adfc27b54a76a05585f3f45bf90/B/a57df684ebd4407e9ca9d68f74a40615 to hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/archive/data/default/TestAcidGuarantees/f0252adfc27b54a76a05585f3f45bf90/B/a57df684ebd4407e9ca9d68f74a40615 2024-11-24T22:14:19,851 DEBUG [StoreCloser-TestAcidGuarantees,,1732486426439.f0252adfc27b54a76a05585f3f45bf90.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/f0252adfc27b54a76a05585f3f45bf90/B/56630cf0374945f290b0ada498ece426 to hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/archive/data/default/TestAcidGuarantees/f0252adfc27b54a76a05585f3f45bf90/B/56630cf0374945f290b0ada498ece426 2024-11-24T22:14:19,852 DEBUG [StoreCloser-TestAcidGuarantees,,1732486426439.f0252adfc27b54a76a05585f3f45bf90.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/f0252adfc27b54a76a05585f3f45bf90/B/f786dc1420844a9da33ca12b1b422ea3 to hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/archive/data/default/TestAcidGuarantees/f0252adfc27b54a76a05585f3f45bf90/B/f786dc1420844a9da33ca12b1b422ea3 2024-11-24T22:14:19,853 DEBUG [StoreCloser-TestAcidGuarantees,,1732486426439.f0252adfc27b54a76a05585f3f45bf90.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/f0252adfc27b54a76a05585f3f45bf90/B/417bf79f9dda4788b24c3865cecc3237 to hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/archive/data/default/TestAcidGuarantees/f0252adfc27b54a76a05585f3f45bf90/B/417bf79f9dda4788b24c3865cecc3237 2024-11-24T22:14:19,855 DEBUG [StoreCloser-TestAcidGuarantees,,1732486426439.f0252adfc27b54a76a05585f3f45bf90.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/f0252adfc27b54a76a05585f3f45bf90/B/7ddf65b17c1948cb8dac4dce2ff012c6 to hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/archive/data/default/TestAcidGuarantees/f0252adfc27b54a76a05585f3f45bf90/B/7ddf65b17c1948cb8dac4dce2ff012c6 2024-11-24T22:14:19,856 DEBUG [StoreCloser-TestAcidGuarantees,,1732486426439.f0252adfc27b54a76a05585f3f45bf90.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/f0252adfc27b54a76a05585f3f45bf90/B/da93b0c6087f4675a21ee70eb6f87149 to hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/archive/data/default/TestAcidGuarantees/f0252adfc27b54a76a05585f3f45bf90/B/da93b0c6087f4675a21ee70eb6f87149 2024-11-24T22:14:19,857 DEBUG [StoreCloser-TestAcidGuarantees,,1732486426439.f0252adfc27b54a76a05585f3f45bf90.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/f0252adfc27b54a76a05585f3f45bf90/B/70abb3621bf148439ad48345232947f0 to hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/archive/data/default/TestAcidGuarantees/f0252adfc27b54a76a05585f3f45bf90/B/70abb3621bf148439ad48345232947f0 2024-11-24T22:14:19,858 DEBUG [StoreCloser-TestAcidGuarantees,,1732486426439.f0252adfc27b54a76a05585f3f45bf90.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/f0252adfc27b54a76a05585f3f45bf90/B/e63699926e974340b82e6c44beff995b to hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/archive/data/default/TestAcidGuarantees/f0252adfc27b54a76a05585f3f45bf90/B/e63699926e974340b82e6c44beff995b 2024-11-24T22:14:19,859 DEBUG [StoreCloser-TestAcidGuarantees,,1732486426439.f0252adfc27b54a76a05585f3f45bf90.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/f0252adfc27b54a76a05585f3f45bf90/B/cc7c51615aeb4494be1ea60782c5134a to hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/archive/data/default/TestAcidGuarantees/f0252adfc27b54a76a05585f3f45bf90/B/cc7c51615aeb4494be1ea60782c5134a 2024-11-24T22:14:19,861 DEBUG [StoreCloser-TestAcidGuarantees,,1732486426439.f0252adfc27b54a76a05585f3f45bf90.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/f0252adfc27b54a76a05585f3f45bf90/B/4d4d753c5b6141dbba6ad07b969acd4b to hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/archive/data/default/TestAcidGuarantees/f0252adfc27b54a76a05585f3f45bf90/B/4d4d753c5b6141dbba6ad07b969acd4b 2024-11-24T22:14:19,862 DEBUG [StoreCloser-TestAcidGuarantees,,1732486426439.f0252adfc27b54a76a05585f3f45bf90.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/f0252adfc27b54a76a05585f3f45bf90/B/13d49dd09c36477fa2ccd8d4b6050657 to hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/archive/data/default/TestAcidGuarantees/f0252adfc27b54a76a05585f3f45bf90/B/13d49dd09c36477fa2ccd8d4b6050657 2024-11-24T22:14:19,863 DEBUG [StoreCloser-TestAcidGuarantees,,1732486426439.f0252adfc27b54a76a05585f3f45bf90.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/f0252adfc27b54a76a05585f3f45bf90/B/60727be57d64464b947b404585a28958 to hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/archive/data/default/TestAcidGuarantees/f0252adfc27b54a76a05585f3f45bf90/B/60727be57d64464b947b404585a28958 2024-11-24T22:14:19,864 DEBUG [StoreCloser-TestAcidGuarantees,,1732486426439.f0252adfc27b54a76a05585f3f45bf90.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/f0252adfc27b54a76a05585f3f45bf90/B/b390b104502d4d5393ffd3f302d7df00 to hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/archive/data/default/TestAcidGuarantees/f0252adfc27b54a76a05585f3f45bf90/B/b390b104502d4d5393ffd3f302d7df00 2024-11-24T22:14:19,865 DEBUG [StoreCloser-TestAcidGuarantees,,1732486426439.f0252adfc27b54a76a05585f3f45bf90.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/f0252adfc27b54a76a05585f3f45bf90/B/81b92b86e15441e090171e98694d5519 to hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/archive/data/default/TestAcidGuarantees/f0252adfc27b54a76a05585f3f45bf90/B/81b92b86e15441e090171e98694d5519 2024-11-24T22:14:19,866 DEBUG [StoreCloser-TestAcidGuarantees,,1732486426439.f0252adfc27b54a76a05585f3f45bf90.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/f0252adfc27b54a76a05585f3f45bf90/B/1496c056f69e420c9011e49538e1d877 to hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/archive/data/default/TestAcidGuarantees/f0252adfc27b54a76a05585f3f45bf90/B/1496c056f69e420c9011e49538e1d877 2024-11-24T22:14:19,867 DEBUG [StoreCloser-TestAcidGuarantees,,1732486426439.f0252adfc27b54a76a05585f3f45bf90.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/f0252adfc27b54a76a05585f3f45bf90/B/6662a3106960477e8a8e7608436590b0 to hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/archive/data/default/TestAcidGuarantees/f0252adfc27b54a76a05585f3f45bf90/B/6662a3106960477e8a8e7608436590b0 2024-11-24T22:14:19,869 DEBUG [StoreCloser-TestAcidGuarantees,,1732486426439.f0252adfc27b54a76a05585f3f45bf90.-1 {}] regionserver.HStore(2317): Moving the files [hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/f0252adfc27b54a76a05585f3f45bf90/C/6cb278bb60bd4562b947a68e3f06567b, hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/f0252adfc27b54a76a05585f3f45bf90/C/cb90eafdaeba493ea99a4ef7d255056f, hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/f0252adfc27b54a76a05585f3f45bf90/C/88c916dc5d6b43af9776f182ebf2959c, hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/f0252adfc27b54a76a05585f3f45bf90/C/c3390572911442069a3598a2de63958d, hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/f0252adfc27b54a76a05585f3f45bf90/C/7fadf24fe8a7459ea252bf9e20a561a7, hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/f0252adfc27b54a76a05585f3f45bf90/C/bc92118f30b44a4995b050c164edd61f, hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/f0252adfc27b54a76a05585f3f45bf90/C/85c936947bd84c1e8e5da34b16ed5bae, hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/f0252adfc27b54a76a05585f3f45bf90/C/7bbbf641f1b44d9195ae936a79cb2684, hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/f0252adfc27b54a76a05585f3f45bf90/C/c4712c657f28446ea38507429ade8164, hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/f0252adfc27b54a76a05585f3f45bf90/C/de4b0d358bae4edb83ce2b00b511adb1, hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/f0252adfc27b54a76a05585f3f45bf90/C/f98021b20f514ebf8abbfa23a71f409c, hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/f0252adfc27b54a76a05585f3f45bf90/C/55c31382a1984b16bbd575e5a6afce7b, hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/f0252adfc27b54a76a05585f3f45bf90/C/2eeb43633aa74ec58c0313ac4a1fc08b, hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/f0252adfc27b54a76a05585f3f45bf90/C/0aa3f38d00954b41978dd2235eb55aa5, hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/f0252adfc27b54a76a05585f3f45bf90/C/e8d52e0fc5344af581b6bdc556bace1e, hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/f0252adfc27b54a76a05585f3f45bf90/C/d1accac28c8541a0b967ecefbff6faa6, hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/f0252adfc27b54a76a05585f3f45bf90/C/984a805e6d2a48d3b398a98cf43f97d2, hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/f0252adfc27b54a76a05585f3f45bf90/C/6b841508335b4fd0b3e088cb97a7ba51, hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/f0252adfc27b54a76a05585f3f45bf90/C/fc8735a74d5548058f328073367c7532, hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/f0252adfc27b54a76a05585f3f45bf90/C/05ada7a4ea8043d293e29bd7672e3405, hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/f0252adfc27b54a76a05585f3f45bf90/C/7c396145845b4013bbcfdadc15e47807, hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/f0252adfc27b54a76a05585f3f45bf90/C/96d7872a2b26416798bfbc2e3cf792d4, hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/f0252adfc27b54a76a05585f3f45bf90/C/8eb7820125494486bd2fd4f8fb01fe21, hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/f0252adfc27b54a76a05585f3f45bf90/C/a92a99c7f4e04ab09097ec22773f3fd9, hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/f0252adfc27b54a76a05585f3f45bf90/C/3e8950a3d04e473f9a791f9f71875f4e, hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/f0252adfc27b54a76a05585f3f45bf90/C/6c8934fad0be457a99fbc833539d7a24, hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/f0252adfc27b54a76a05585f3f45bf90/C/830623b4e53f462483bc52b3a55ecc37, hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/f0252adfc27b54a76a05585f3f45bf90/C/017690ed58eb4d15a0a1ed73bc08a541, hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/f0252adfc27b54a76a05585f3f45bf90/C/94d5272cd82b4ec994e3c472dd9e27f2, hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/f0252adfc27b54a76a05585f3f45bf90/C/3fec91296d704444acf8427c12e1b565] to archive 2024-11-24T22:14:19,870 DEBUG [StoreCloser-TestAcidGuarantees,,1732486426439.f0252adfc27b54a76a05585f3f45bf90.-1 {}] backup.HFileArchiver(360): Archiving compacted files. 2024-11-24T22:14:19,872 DEBUG [StoreCloser-TestAcidGuarantees,,1732486426439.f0252adfc27b54a76a05585f3f45bf90.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/f0252adfc27b54a76a05585f3f45bf90/C/6cb278bb60bd4562b947a68e3f06567b to hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/archive/data/default/TestAcidGuarantees/f0252adfc27b54a76a05585f3f45bf90/C/6cb278bb60bd4562b947a68e3f06567b 2024-11-24T22:14:19,873 DEBUG [StoreCloser-TestAcidGuarantees,,1732486426439.f0252adfc27b54a76a05585f3f45bf90.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/f0252adfc27b54a76a05585f3f45bf90/C/cb90eafdaeba493ea99a4ef7d255056f to hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/archive/data/default/TestAcidGuarantees/f0252adfc27b54a76a05585f3f45bf90/C/cb90eafdaeba493ea99a4ef7d255056f 2024-11-24T22:14:19,875 DEBUG [StoreCloser-TestAcidGuarantees,,1732486426439.f0252adfc27b54a76a05585f3f45bf90.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/f0252adfc27b54a76a05585f3f45bf90/C/88c916dc5d6b43af9776f182ebf2959c to hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/archive/data/default/TestAcidGuarantees/f0252adfc27b54a76a05585f3f45bf90/C/88c916dc5d6b43af9776f182ebf2959c 2024-11-24T22:14:19,876 DEBUG [StoreCloser-TestAcidGuarantees,,1732486426439.f0252adfc27b54a76a05585f3f45bf90.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/f0252adfc27b54a76a05585f3f45bf90/C/c3390572911442069a3598a2de63958d to hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/archive/data/default/TestAcidGuarantees/f0252adfc27b54a76a05585f3f45bf90/C/c3390572911442069a3598a2de63958d 2024-11-24T22:14:19,877 DEBUG [StoreCloser-TestAcidGuarantees,,1732486426439.f0252adfc27b54a76a05585f3f45bf90.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/f0252adfc27b54a76a05585f3f45bf90/C/7fadf24fe8a7459ea252bf9e20a561a7 to hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/archive/data/default/TestAcidGuarantees/f0252adfc27b54a76a05585f3f45bf90/C/7fadf24fe8a7459ea252bf9e20a561a7 2024-11-24T22:14:19,878 DEBUG [StoreCloser-TestAcidGuarantees,,1732486426439.f0252adfc27b54a76a05585f3f45bf90.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/f0252adfc27b54a76a05585f3f45bf90/C/bc92118f30b44a4995b050c164edd61f to hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/archive/data/default/TestAcidGuarantees/f0252adfc27b54a76a05585f3f45bf90/C/bc92118f30b44a4995b050c164edd61f 2024-11-24T22:14:19,880 DEBUG [StoreCloser-TestAcidGuarantees,,1732486426439.f0252adfc27b54a76a05585f3f45bf90.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/f0252adfc27b54a76a05585f3f45bf90/C/85c936947bd84c1e8e5da34b16ed5bae to hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/archive/data/default/TestAcidGuarantees/f0252adfc27b54a76a05585f3f45bf90/C/85c936947bd84c1e8e5da34b16ed5bae 2024-11-24T22:14:19,881 DEBUG [StoreCloser-TestAcidGuarantees,,1732486426439.f0252adfc27b54a76a05585f3f45bf90.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/f0252adfc27b54a76a05585f3f45bf90/C/7bbbf641f1b44d9195ae936a79cb2684 to hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/archive/data/default/TestAcidGuarantees/f0252adfc27b54a76a05585f3f45bf90/C/7bbbf641f1b44d9195ae936a79cb2684 2024-11-24T22:14:19,882 DEBUG [StoreCloser-TestAcidGuarantees,,1732486426439.f0252adfc27b54a76a05585f3f45bf90.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/f0252adfc27b54a76a05585f3f45bf90/C/c4712c657f28446ea38507429ade8164 to hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/archive/data/default/TestAcidGuarantees/f0252adfc27b54a76a05585f3f45bf90/C/c4712c657f28446ea38507429ade8164 2024-11-24T22:14:19,884 DEBUG [StoreCloser-TestAcidGuarantees,,1732486426439.f0252adfc27b54a76a05585f3f45bf90.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/f0252adfc27b54a76a05585f3f45bf90/C/de4b0d358bae4edb83ce2b00b511adb1 to hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/archive/data/default/TestAcidGuarantees/f0252adfc27b54a76a05585f3f45bf90/C/de4b0d358bae4edb83ce2b00b511adb1 2024-11-24T22:14:19,885 DEBUG [StoreCloser-TestAcidGuarantees,,1732486426439.f0252adfc27b54a76a05585f3f45bf90.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/f0252adfc27b54a76a05585f3f45bf90/C/f98021b20f514ebf8abbfa23a71f409c to hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/archive/data/default/TestAcidGuarantees/f0252adfc27b54a76a05585f3f45bf90/C/f98021b20f514ebf8abbfa23a71f409c 2024-11-24T22:14:19,886 DEBUG [StoreCloser-TestAcidGuarantees,,1732486426439.f0252adfc27b54a76a05585f3f45bf90.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/f0252adfc27b54a76a05585f3f45bf90/C/55c31382a1984b16bbd575e5a6afce7b to hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/archive/data/default/TestAcidGuarantees/f0252adfc27b54a76a05585f3f45bf90/C/55c31382a1984b16bbd575e5a6afce7b 2024-11-24T22:14:19,887 DEBUG [StoreCloser-TestAcidGuarantees,,1732486426439.f0252adfc27b54a76a05585f3f45bf90.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/f0252adfc27b54a76a05585f3f45bf90/C/2eeb43633aa74ec58c0313ac4a1fc08b to hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/archive/data/default/TestAcidGuarantees/f0252adfc27b54a76a05585f3f45bf90/C/2eeb43633aa74ec58c0313ac4a1fc08b 2024-11-24T22:14:19,888 DEBUG [StoreCloser-TestAcidGuarantees,,1732486426439.f0252adfc27b54a76a05585f3f45bf90.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/f0252adfc27b54a76a05585f3f45bf90/C/0aa3f38d00954b41978dd2235eb55aa5 to hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/archive/data/default/TestAcidGuarantees/f0252adfc27b54a76a05585f3f45bf90/C/0aa3f38d00954b41978dd2235eb55aa5 2024-11-24T22:14:19,889 DEBUG [StoreCloser-TestAcidGuarantees,,1732486426439.f0252adfc27b54a76a05585f3f45bf90.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/f0252adfc27b54a76a05585f3f45bf90/C/e8d52e0fc5344af581b6bdc556bace1e to hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/archive/data/default/TestAcidGuarantees/f0252adfc27b54a76a05585f3f45bf90/C/e8d52e0fc5344af581b6bdc556bace1e 2024-11-24T22:14:19,890 DEBUG [StoreCloser-TestAcidGuarantees,,1732486426439.f0252adfc27b54a76a05585f3f45bf90.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/f0252adfc27b54a76a05585f3f45bf90/C/d1accac28c8541a0b967ecefbff6faa6 to hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/archive/data/default/TestAcidGuarantees/f0252adfc27b54a76a05585f3f45bf90/C/d1accac28c8541a0b967ecefbff6faa6 2024-11-24T22:14:19,891 DEBUG [StoreCloser-TestAcidGuarantees,,1732486426439.f0252adfc27b54a76a05585f3f45bf90.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/f0252adfc27b54a76a05585f3f45bf90/C/984a805e6d2a48d3b398a98cf43f97d2 to hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/archive/data/default/TestAcidGuarantees/f0252adfc27b54a76a05585f3f45bf90/C/984a805e6d2a48d3b398a98cf43f97d2 2024-11-24T22:14:19,892 DEBUG [StoreCloser-TestAcidGuarantees,,1732486426439.f0252adfc27b54a76a05585f3f45bf90.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/f0252adfc27b54a76a05585f3f45bf90/C/6b841508335b4fd0b3e088cb97a7ba51 to hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/archive/data/default/TestAcidGuarantees/f0252adfc27b54a76a05585f3f45bf90/C/6b841508335b4fd0b3e088cb97a7ba51 2024-11-24T22:14:19,893 DEBUG [StoreCloser-TestAcidGuarantees,,1732486426439.f0252adfc27b54a76a05585f3f45bf90.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/f0252adfc27b54a76a05585f3f45bf90/C/fc8735a74d5548058f328073367c7532 to hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/archive/data/default/TestAcidGuarantees/f0252adfc27b54a76a05585f3f45bf90/C/fc8735a74d5548058f328073367c7532 2024-11-24T22:14:19,894 DEBUG [StoreCloser-TestAcidGuarantees,,1732486426439.f0252adfc27b54a76a05585f3f45bf90.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/f0252adfc27b54a76a05585f3f45bf90/C/05ada7a4ea8043d293e29bd7672e3405 to hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/archive/data/default/TestAcidGuarantees/f0252adfc27b54a76a05585f3f45bf90/C/05ada7a4ea8043d293e29bd7672e3405 2024-11-24T22:14:19,895 DEBUG [StoreCloser-TestAcidGuarantees,,1732486426439.f0252adfc27b54a76a05585f3f45bf90.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/f0252adfc27b54a76a05585f3f45bf90/C/7c396145845b4013bbcfdadc15e47807 to hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/archive/data/default/TestAcidGuarantees/f0252adfc27b54a76a05585f3f45bf90/C/7c396145845b4013bbcfdadc15e47807 2024-11-24T22:14:19,896 DEBUG [StoreCloser-TestAcidGuarantees,,1732486426439.f0252adfc27b54a76a05585f3f45bf90.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/f0252adfc27b54a76a05585f3f45bf90/C/96d7872a2b26416798bfbc2e3cf792d4 to hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/archive/data/default/TestAcidGuarantees/f0252adfc27b54a76a05585f3f45bf90/C/96d7872a2b26416798bfbc2e3cf792d4 2024-11-24T22:14:19,897 DEBUG [StoreCloser-TestAcidGuarantees,,1732486426439.f0252adfc27b54a76a05585f3f45bf90.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/f0252adfc27b54a76a05585f3f45bf90/C/8eb7820125494486bd2fd4f8fb01fe21 to hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/archive/data/default/TestAcidGuarantees/f0252adfc27b54a76a05585f3f45bf90/C/8eb7820125494486bd2fd4f8fb01fe21 2024-11-24T22:14:19,898 DEBUG [StoreCloser-TestAcidGuarantees,,1732486426439.f0252adfc27b54a76a05585f3f45bf90.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/f0252adfc27b54a76a05585f3f45bf90/C/a92a99c7f4e04ab09097ec22773f3fd9 to hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/archive/data/default/TestAcidGuarantees/f0252adfc27b54a76a05585f3f45bf90/C/a92a99c7f4e04ab09097ec22773f3fd9 2024-11-24T22:14:19,899 DEBUG [StoreCloser-TestAcidGuarantees,,1732486426439.f0252adfc27b54a76a05585f3f45bf90.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/f0252adfc27b54a76a05585f3f45bf90/C/3e8950a3d04e473f9a791f9f71875f4e to hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/archive/data/default/TestAcidGuarantees/f0252adfc27b54a76a05585f3f45bf90/C/3e8950a3d04e473f9a791f9f71875f4e 2024-11-24T22:14:19,900 DEBUG [StoreCloser-TestAcidGuarantees,,1732486426439.f0252adfc27b54a76a05585f3f45bf90.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/f0252adfc27b54a76a05585f3f45bf90/C/6c8934fad0be457a99fbc833539d7a24 to hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/archive/data/default/TestAcidGuarantees/f0252adfc27b54a76a05585f3f45bf90/C/6c8934fad0be457a99fbc833539d7a24 2024-11-24T22:14:19,901 DEBUG [StoreCloser-TestAcidGuarantees,,1732486426439.f0252adfc27b54a76a05585f3f45bf90.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/f0252adfc27b54a76a05585f3f45bf90/C/830623b4e53f462483bc52b3a55ecc37 to hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/archive/data/default/TestAcidGuarantees/f0252adfc27b54a76a05585f3f45bf90/C/830623b4e53f462483bc52b3a55ecc37 2024-11-24T22:14:19,901 DEBUG [StoreCloser-TestAcidGuarantees,,1732486426439.f0252adfc27b54a76a05585f3f45bf90.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/f0252adfc27b54a76a05585f3f45bf90/C/017690ed58eb4d15a0a1ed73bc08a541 to hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/archive/data/default/TestAcidGuarantees/f0252adfc27b54a76a05585f3f45bf90/C/017690ed58eb4d15a0a1ed73bc08a541 2024-11-24T22:14:19,902 DEBUG [StoreCloser-TestAcidGuarantees,,1732486426439.f0252adfc27b54a76a05585f3f45bf90.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/f0252adfc27b54a76a05585f3f45bf90/C/94d5272cd82b4ec994e3c472dd9e27f2 to hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/archive/data/default/TestAcidGuarantees/f0252adfc27b54a76a05585f3f45bf90/C/94d5272cd82b4ec994e3c472dd9e27f2 2024-11-24T22:14:19,903 DEBUG [StoreCloser-TestAcidGuarantees,,1732486426439.f0252adfc27b54a76a05585f3f45bf90.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/f0252adfc27b54a76a05585f3f45bf90/C/3fec91296d704444acf8427c12e1b565 to hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/archive/data/default/TestAcidGuarantees/f0252adfc27b54a76a05585f3f45bf90/C/3fec91296d704444acf8427c12e1b565 2024-11-24T22:14:19,908 DEBUG [RS_CLOSE_REGION-regionserver/bd6592c37bb2:0-0 {event_type=M_RS_CLOSE_REGION, pid=28}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/f0252adfc27b54a76a05585f3f45bf90/recovered.edits/734.seqid, newMaxSeqId=734, maxSeqId=1 2024-11-24T22:14:19,909 INFO [RS_CLOSE_REGION-regionserver/bd6592c37bb2:0-0 {event_type=M_RS_CLOSE_REGION, pid=28}] regionserver.HRegion(1973): Closed TestAcidGuarantees,,1732486426439.f0252adfc27b54a76a05585f3f45bf90. 2024-11-24T22:14:19,909 DEBUG [RS_CLOSE_REGION-regionserver/bd6592c37bb2:0-0 {event_type=M_RS_CLOSE_REGION, pid=28}] regionserver.HRegion(1676): Region close journal for f0252adfc27b54a76a05585f3f45bf90: Waiting for close lock at 1732486458514Running coprocessor pre-close hooks at 1732486458514Disabling compacts and flushes for region at 1732486458514Disabling writes for close at 1732486458515 (+1 ms)Obtaining lock to block concurrent updates at 1732486458515Preparing flush snapshotting stores in f0252adfc27b54a76a05585f3f45bf90 at 1732486458515Finished memstore snapshotting TestAcidGuarantees,,1732486426439.f0252adfc27b54a76a05585f3f45bf90., syncing WAL and waiting on mvcc, flushsize=dataSize=20610, getHeapSize=61920, getOffHeapSize=0, getCellsCount=450 at 1732486458515Flushing stores of TestAcidGuarantees,,1732486426439.f0252adfc27b54a76a05585f3f45bf90. at 1732486458516 (+1 ms)Flushing f0252adfc27b54a76a05585f3f45bf90/A: creating writer at 1732486458516Flushing f0252adfc27b54a76a05585f3f45bf90/A: appending metadata at 1732486458519 (+3 ms)Flushing f0252adfc27b54a76a05585f3f45bf90/A: closing flushed file at 1732486458519Flushing f0252adfc27b54a76a05585f3f45bf90/B: creating writer at 1732486458930 (+411 ms)Flushing f0252adfc27b54a76a05585f3f45bf90/B: appending metadata at 1732486458933 (+3 ms)Flushing f0252adfc27b54a76a05585f3f45bf90/B: closing flushed file at 1732486458933Flushing f0252adfc27b54a76a05585f3f45bf90/C: creating writer at 1732486459342 (+409 ms)Flushing f0252adfc27b54a76a05585f3f45bf90/C: appending metadata at 1732486459345 (+3 ms)Flushing f0252adfc27b54a76a05585f3f45bf90/C: closing flushed file at 1732486459345Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@135d11cc: reopening flushed file at 1732486459755 (+410 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@56e40317: reopening flushed file at 1732486459761 (+6 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@654de3d8: reopening flushed file at 1732486459765 (+4 ms)Finished flush of dataSize ~20.13 KB/20610, heapSize ~60.47 KB/61920, currentSize=0 B/0 for f0252adfc27b54a76a05585f3f45bf90 in 1255ms, sequenceid=731, compaction requested=true at 1732486459770 (+5 ms)Writing region close event to WAL at 1732486459905 (+135 ms)Running coprocessor post-close hooks at 1732486459909 (+4 ms)Closed at 1732486459909 2024-11-24T22:14:19,912 INFO [RS_CLOSE_REGION-regionserver/bd6592c37bb2:0-0 {event_type=M_RS_CLOSE_REGION, pid=28}] handler.UnassignRegionHandler(157): Closed f0252adfc27b54a76a05585f3f45bf90 2024-11-24T22:14:19,912 INFO [PEWorker-4 {}] assignment.RegionStateStore(223): pid=27 updating hbase:meta row=f0252adfc27b54a76a05585f3f45bf90, regionState=CLOSED 2024-11-24T22:14:19,914 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=28, ppid=27, state=RUNNABLE, hasLock=false; CloseRegionProcedure f0252adfc27b54a76a05585f3f45bf90, server=bd6592c37bb2,40087,1732486423713 because future has completed 2024-11-24T22:14:19,917 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=28, resume processing ppid=27 2024-11-24T22:14:19,917 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=28, ppid=27, state=SUCCESS, hasLock=false; CloseRegionProcedure f0252adfc27b54a76a05585f3f45bf90, server=bd6592c37bb2,40087,1732486423713 in 1.5580 sec 2024-11-24T22:14:19,919 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=27, resume processing ppid=26 2024-11-24T22:14:19,919 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=27, ppid=26, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=f0252adfc27b54a76a05585f3f45bf90, UNASSIGN in 1.5650 sec 2024-11-24T22:14:19,921 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=26, resume processing ppid=25 2024-11-24T22:14:19,921 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=26, ppid=25, state=SUCCESS, hasLock=false; CloseTableRegionsProcedure table=TestAcidGuarantees in 1.5720 sec 2024-11-24T22:14:19,923 DEBUG [PEWorker-5 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1732486459922"}]},"ts":"1732486459922"} 2024-11-24T22:14:19,924 INFO [PEWorker-5 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=DISABLED in hbase:meta 2024-11-24T22:14:19,924 INFO [PEWorker-5 {}] procedure.DisableTableProcedure(296): Set TestAcidGuarantees to state=DISABLED 2024-11-24T22:14:19,926 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=25, state=SUCCESS, hasLock=false; DisableTableProcedure table=TestAcidGuarantees in 1.5860 sec 2024-11-24T22:14:20,491 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37701 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=25 2024-11-24T22:14:20,491 INFO [RPCClient-NioEventLoopGroup-4-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: DISABLE, Table Name: default:TestAcidGuarantees completed 2024-11-24T22:14:20,494 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37701 {}] master.HMaster$5(2570): Client=jenkins//172.17.0.2 delete TestAcidGuarantees 2024-11-24T22:14:20,499 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37701 {}] procedure2.ProcedureExecutor(1139): Stored pid=29, state=RUNNABLE:DELETE_TABLE_PRE_OPERATION, hasLock=false; DeleteTableProcedure table=TestAcidGuarantees 2024-11-24T22:14:20,500 DEBUG [PEWorker-4 {}] procedure.DeleteTableProcedure(103): Waiting for RIT for pid=29, state=RUNNABLE:DELETE_TABLE_PRE_OPERATION, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-24T22:14:20,501 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37701 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=29 2024-11-24T22:14:20,501 DEBUG [PEWorker-4 {}] procedure.DeleteTableProcedure(115): Deleting regions from filesystem for pid=29, state=RUNNABLE:DELETE_TABLE_CLEAR_FS_LAYOUT, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-24T22:14:20,506 DEBUG [HFileArchiver-1 {}] backup.HFileArchiver(131): ARCHIVING hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/f0252adfc27b54a76a05585f3f45bf90 2024-11-24T22:14:20,509 DEBUG [HFileArchiver-1 {}] backup.HFileArchiver(159): Archiving [FileablePath, hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/f0252adfc27b54a76a05585f3f45bf90/A, FileablePath, hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/f0252adfc27b54a76a05585f3f45bf90/B, FileablePath, hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/f0252adfc27b54a76a05585f3f45bf90/C, FileablePath, hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/f0252adfc27b54a76a05585f3f45bf90/recovered.edits] 2024-11-24T22:14:20,512 DEBUG [HFileArchiver-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/f0252adfc27b54a76a05585f3f45bf90/A/10d027912dea4ec1ad5bae02c41dea6e to hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/archive/data/default/TestAcidGuarantees/f0252adfc27b54a76a05585f3f45bf90/A/10d027912dea4ec1ad5bae02c41dea6e 2024-11-24T22:14:20,514 DEBUG [HFileArchiver-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/f0252adfc27b54a76a05585f3f45bf90/A/c30a602de1784cbdba4e4748b7be0e24 to hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/archive/data/default/TestAcidGuarantees/f0252adfc27b54a76a05585f3f45bf90/A/c30a602de1784cbdba4e4748b7be0e24 2024-11-24T22:14:20,515 DEBUG [HFileArchiver-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/f0252adfc27b54a76a05585f3f45bf90/A/f140922ff681458cb1d780bb282b0389 to hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/archive/data/default/TestAcidGuarantees/f0252adfc27b54a76a05585f3f45bf90/A/f140922ff681458cb1d780bb282b0389 2024-11-24T22:14:20,517 DEBUG [HFileArchiver-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/f0252adfc27b54a76a05585f3f45bf90/B/1abd35719ed74cc2a850acc71d60c279 to hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/archive/data/default/TestAcidGuarantees/f0252adfc27b54a76a05585f3f45bf90/B/1abd35719ed74cc2a850acc71d60c279 2024-11-24T22:14:20,518 DEBUG [HFileArchiver-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/f0252adfc27b54a76a05585f3f45bf90/B/37bf2eb16dda4726b76dda8535c368bf to hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/archive/data/default/TestAcidGuarantees/f0252adfc27b54a76a05585f3f45bf90/B/37bf2eb16dda4726b76dda8535c368bf 2024-11-24T22:14:20,520 DEBUG [HFileArchiver-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/f0252adfc27b54a76a05585f3f45bf90/B/fbe576cfb9064721b6aa3a1b5d3c25f2 to hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/archive/data/default/TestAcidGuarantees/f0252adfc27b54a76a05585f3f45bf90/B/fbe576cfb9064721b6aa3a1b5d3c25f2 2024-11-24T22:14:20,522 DEBUG [HFileArchiver-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/f0252adfc27b54a76a05585f3f45bf90/C/02e6805bcf5747a79909959b2cfa0786 to hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/archive/data/default/TestAcidGuarantees/f0252adfc27b54a76a05585f3f45bf90/C/02e6805bcf5747a79909959b2cfa0786 2024-11-24T22:14:20,523 DEBUG [HFileArchiver-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/f0252adfc27b54a76a05585f3f45bf90/C/61ab401cfaa3465fb546f51638a9b0ea to hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/archive/data/default/TestAcidGuarantees/f0252adfc27b54a76a05585f3f45bf90/C/61ab401cfaa3465fb546f51638a9b0ea 2024-11-24T22:14:20,524 DEBUG [HFileArchiver-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/f0252adfc27b54a76a05585f3f45bf90/C/cc09dcf327b046d197b0e6a007e8c84d to hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/archive/data/default/TestAcidGuarantees/f0252adfc27b54a76a05585f3f45bf90/C/cc09dcf327b046d197b0e6a007e8c84d 2024-11-24T22:14:20,527 DEBUG [HFileArchiver-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/f0252adfc27b54a76a05585f3f45bf90/recovered.edits/734.seqid to hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/archive/data/default/TestAcidGuarantees/f0252adfc27b54a76a05585f3f45bf90/recovered.edits/734.seqid 2024-11-24T22:14:20,527 DEBUG [HFileArchiver-1 {}] backup.HFileArchiver(610): Deleted hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/f0252adfc27b54a76a05585f3f45bf90 2024-11-24T22:14:20,527 DEBUG [PEWorker-4 {}] procedure.DeleteTableProcedure(313): Archived TestAcidGuarantees regions 2024-11-24T22:14:20,532 DEBUG [PEWorker-4 {}] procedure.DeleteTableProcedure(120): Deleting regions from META for pid=29, state=RUNNABLE:DELETE_TABLE_REMOVE_FROM_META, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-24T22:14:20,536 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=40087 {}] util.ReflectedFunctionCache(97): Populated cache for org.apache.hadoop.hbase.filter.KeyOnlyFilter in 0ms 2024-11-24T22:14:20,541 WARN [PEWorker-4 {}] procedure.DeleteTableProcedure(370): Deleting some vestigial 1 rows of TestAcidGuarantees from hbase:meta 2024-11-24T22:14:20,571 DEBUG [PEWorker-4 {}] procedure.DeleteTableProcedure(407): Removing 'TestAcidGuarantees' descriptor. 2024-11-24T22:14:20,573 DEBUG [PEWorker-4 {}] procedure.DeleteTableProcedure(126): Deleting assignment state for pid=29, state=RUNNABLE:DELETE_TABLE_UNASSIGN_REGIONS, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-24T22:14:20,573 DEBUG [PEWorker-4 {}] procedure.DeleteTableProcedure(397): Removing 'TestAcidGuarantees' from region states. 2024-11-24T22:14:20,573 DEBUG [PEWorker-4 {}] assignment.RegionStateStore(723): Delete {"totalColumns":1,"row":"TestAcidGuarantees,,1732486426439.f0252adfc27b54a76a05585f3f45bf90.","families":{"info":[{"qualifier":"","vlen":0,"tag":[],"timestamp":"1732486460573"}]},"ts":"9223372036854775807"} 2024-11-24T22:14:20,578 INFO [PEWorker-4 {}] assignment.RegionStateStore(562): Deleted 1 regions from META 2024-11-24T22:14:20,578 DEBUG [PEWorker-4 {}] assignment.RegionStateStore(563): Deleted regions: [{ENCODED => f0252adfc27b54a76a05585f3f45bf90, NAME => 'TestAcidGuarantees,,1732486426439.f0252adfc27b54a76a05585f3f45bf90.', STARTKEY => '', ENDKEY => ''}] 2024-11-24T22:14:20,578 DEBUG [PEWorker-4 {}] procedure.DeleteTableProcedure(401): Marking 'TestAcidGuarantees' as deleted. 2024-11-24T22:14:20,578 DEBUG [PEWorker-4 {}] hbase.MetaTableAccessor(964): Delete {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":0,"tag":[],"timestamp":"1732486460578"}]},"ts":"9223372036854775807"} 2024-11-24T22:14:20,581 INFO [PEWorker-4 {}] hbase.MetaTableAccessor(867): Deleted table TestAcidGuarantees state from META 2024-11-24T22:14:20,582 DEBUG [PEWorker-4 {}] procedure.DeleteTableProcedure(133): Finished pid=29, state=RUNNABLE:DELETE_TABLE_POST_OPERATION, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-24T22:14:20,583 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=29, state=SUCCESS, hasLock=false; DeleteTableProcedure table=TestAcidGuarantees in 87 msec 2024-11-24T22:14:20,611 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37701 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=29 2024-11-24T22:14:20,612 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocator(219): Clear meta cache for TestAcidGuarantees 2024-11-24T22:14:20,612 INFO [RPCClient-NioEventLoopGroup-4-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: DELETE, Table Name: default:TestAcidGuarantees completed 2024-11-24T22:14:20,623 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: TestAcidGuaranteesWithEagerPolicy#testMixedAtomicity Thread=247 (was 211) Potentially hanging thread: RPCClient-NioEventLoopGroup-4-6 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RPCClient-NioEventLoopGroup-4-12 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RPCClient-NioEventLoopGroup-4-4 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:62) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:883) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RS_IN_MEMORY_COMPACTION-regionserver/bd6592c37bb2:0-9 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RS_IN_MEMORY_COMPACTION-regionserver/bd6592c37bb2:0-6 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RPCClient-NioEventLoopGroup-4-8 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RPCClient-NioEventLoopGroup-4-11 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RS_IN_MEMORY_COMPACTION-regionserver/bd6592c37bb2:0-0 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RS_CLOSE_REGION-regionserver/bd6592c37bb2:0-0 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RS_IN_MEMORY_COMPACTION-regionserver/bd6592c37bb2:0-3 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RS_IN_MEMORY_COMPACTION-regionserver/bd6592c37bb2:0-4 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RPCClient-NioEventLoopGroup-4-5 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RPCClient-NioEventLoopGroup-4-15 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RS_IN_MEMORY_COMPACTION-regionserver/bd6592c37bb2:0-5 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RS_OPEN_REGION-regionserver/bd6592c37bb2:0-0 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RPCClient-NioEventLoopGroup-4-16 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RPCClient-NioEventLoopGroup-4-7 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: ProcedureExecutor-Async-Task-Executor-2 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RPCClient-NioEventLoopGroup-4-14 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RS_IN_MEMORY_COMPACTION-regionserver/bd6592c37bb2:0-7 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: Timer for 'HBase' metrics system java.base@17.0.11/java.lang.Object.wait(Native Method) java.base@17.0.11/java.util.TimerThread.mainLoop(Timer.java:563) java.base@17.0.11/java.util.TimerThread.run(Timer.java:516) Potentially hanging thread: RPCClient-NioEventLoopGroup-4-10 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RS_IN_MEMORY_COMPACTION-regionserver/bd6592c37bb2:0-2 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RS_FLUSH_OPERATIONS-regionserver/bd6592c37bb2:0-1 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RSProcedureDispatcher-pool-1 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RS_IN_MEMORY_COMPACTION-regionserver/bd6592c37bb2:0-8 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: HFileArchiver-1 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RS_FLUSH_OPERATIONS-regionserver/bd6592c37bb2:0-0 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RS_FLUSH_OPERATIONS-regionserver/bd6592c37bb2:0-2 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RPCClient-NioEventLoopGroup-4-9 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RSProcedureDispatcher-pool-2 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: ProcedureExecutor-Async-Task-Executor-1 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RPCClient-NioEventLoopGroup-4-13 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RS_IN_MEMORY_COMPACTION-regionserver/bd6592c37bb2:0-1 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: ProcedureExecutor-Async-Task-Executor-0 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RS:0;bd6592c37bb2:40087-shortCompactions-0 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.PriorityBlockingQueue.take(PriorityBlockingQueue.java:535) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_-1699160809_22 at /127.0.0.1:44202 [Waiting for operation #13] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: ProcedureExecutor-Async-Task-Executor-3 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) - Thread LEAK? -, OpenFileDescriptor=445 (was 441) - OpenFileDescriptor LEAK? -, MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=347 (was 301) - SystemLoadAverage LEAK? -, ProcessCount=11 (was 11), AvailableMemoryMB=7040 (was 6857) - AvailableMemoryMB LEAK? - 2024-11-24T22:14:20,632 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: TestAcidGuaranteesWithEagerPolicy#testMobMixedAtomicity Thread=247, OpenFileDescriptor=445, MaxFileDescriptor=1048576, SystemLoadAverage=347, ProcessCount=11, AvailableMemoryMB=7040 2024-11-24T22:14:20,633 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37701 {}] util.TableDescriptorChecker(321): MEMSTORE_FLUSHSIZE for table descriptor or "hbase.hregion.memstore.flush.size" (131072) is too small, which might cause very frequent flushing. 2024-11-24T22:14:20,634 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37701 {}] master.HMaster$4(2454): Client=jenkins//172.17.0.2 create 'TestAcidGuarantees', {TABLE_ATTRIBUTES => {METADATA => {'hbase.hregion.compacting.memstore.type' => 'EAGER', 'hbase.hregion.memstore.mslab.enabled' => 'false', 'hbase.memstore.inmemoryflush.threshold.factor' => '0.9'}}}, {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-11-24T22:14:20,635 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37701 {}] procedure2.ProcedureExecutor(1139): Stored pid=30, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=false; CreateTableProcedure table=TestAcidGuarantees 2024-11-24T22:14:20,636 INFO [PEWorker-1 {}] procedure.CreateTableProcedure(89): pid=30, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_PRE_OPERATION 2024-11-24T22:14:20,636 DEBUG [PEWorker-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T22:14:20,636 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37701 {}] master.MasterRpcServices(787): Client=jenkins//172.17.0.2 procedure request for creating table: namespace: "default" qualifier: "TestAcidGuarantees" procId is: 30 2024-11-24T22:14:20,637 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37701 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=30 2024-11-24T22:14:20,637 INFO [PEWorker-1 {}] procedure.CreateTableProcedure(89): pid=30, state=RUNNABLE:CREATE_TABLE_WRITE_FS_LAYOUT, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_WRITE_FS_LAYOUT 2024-11-24T22:14:20,642 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45929 is added to blk_1073741936_1112 (size=1061) 2024-11-24T22:14:20,741 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37701 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=30 2024-11-24T22:14:20,951 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37701 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=30 2024-11-24T22:14:21,045 INFO [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(7572): creating {ENCODED => 63842f73863e9e7b9587486b122e6aae, NAME => 'TestAcidGuarantees,,1732486460633.63842f73863e9e7b9587486b122e6aae.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='TestAcidGuarantees', {TABLE_ATTRIBUTES => {METADATA => {'hbase.hregion.compacting.memstore.type' => 'EAGER', 'hbase.hregion.memstore.mslab.enabled' => 'false', 'hbase.memstore.inmemoryflush.threshold.factor' => '0.9', 'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc 2024-11-24T22:14:21,050 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45929 is added to blk_1073741937_1113 (size=53) 2024-11-24T22:14:21,261 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37701 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=30 2024-11-24T22:14:21,451 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(898): Instantiated TestAcidGuarantees,,1732486460633.63842f73863e9e7b9587486b122e6aae.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-24T22:14:21,452 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1722): Closing 63842f73863e9e7b9587486b122e6aae, disabling compactions & flushes 2024-11-24T22:14:21,452 INFO [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1755): Closing region TestAcidGuarantees,,1732486460633.63842f73863e9e7b9587486b122e6aae. 2024-11-24T22:14:21,452 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1776): Time limited wait for close lock on TestAcidGuarantees,,1732486460633.63842f73863e9e7b9587486b122e6aae. 2024-11-24T22:14:21,452 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1843): Acquired close lock on TestAcidGuarantees,,1732486460633.63842f73863e9e7b9587486b122e6aae. after waiting 0 ms 2024-11-24T22:14:21,452 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1853): Updates disabled for region TestAcidGuarantees,,1732486460633.63842f73863e9e7b9587486b122e6aae. 2024-11-24T22:14:21,452 INFO [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1973): Closed TestAcidGuarantees,,1732486460633.63842f73863e9e7b9587486b122e6aae. 2024-11-24T22:14:21,452 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1676): Region close journal for 63842f73863e9e7b9587486b122e6aae: Waiting for close lock at 1732486461452Disabling compacts and flushes for region at 1732486461452Disabling writes for close at 1732486461452Writing region close event to WAL at 1732486461452Closed at 1732486461452 2024-11-24T22:14:21,453 INFO [PEWorker-1 {}] procedure.CreateTableProcedure(89): pid=30, state=RUNNABLE:CREATE_TABLE_ADD_TO_META, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_ADD_TO_META 2024-11-24T22:14:21,454 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":2,"row":"TestAcidGuarantees,,1732486460633.63842f73863e9e7b9587486b122e6aae.","families":{"info":[{"qualifier":"regioninfo","vlen":52,"tag":[],"timestamp":"1732486461453"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1732486461453"}]},"ts":"1732486461453"} 2024-11-24T22:14:21,456 INFO [PEWorker-1 {}] hbase.MetaTableAccessor(832): Added 1 regions to meta. 2024-11-24T22:14:21,457 INFO [PEWorker-1 {}] procedure.CreateTableProcedure(89): pid=30, state=RUNNABLE:CREATE_TABLE_ASSIGN_REGIONS, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_ASSIGN_REGIONS 2024-11-24T22:14:21,457 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1732486461457"}]},"ts":"1732486461457"} 2024-11-24T22:14:21,459 INFO [PEWorker-1 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=ENABLING in hbase:meta 2024-11-24T22:14:21,460 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=31, ppid=30, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=63842f73863e9e7b9587486b122e6aae, ASSIGN}] 2024-11-24T22:14:21,461 INFO [PEWorker-2 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=31, ppid=30, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=63842f73863e9e7b9587486b122e6aae, ASSIGN 2024-11-24T22:14:21,462 INFO [PEWorker-2 {}] assignment.TransitRegionStateProcedure(269): Starting pid=31, ppid=30, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=63842f73863e9e7b9587486b122e6aae, ASSIGN; state=OFFLINE, location=bd6592c37bb2,40087,1732486423713; forceNewPlan=false, retain=false 2024-11-24T22:14:21,612 INFO [PEWorker-3 {}] assignment.RegionStateStore(223): pid=31 updating hbase:meta row=63842f73863e9e7b9587486b122e6aae, regionState=OPENING, regionLocation=bd6592c37bb2,40087,1732486423713 2024-11-24T22:14:21,615 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=31, ppid=30, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=63842f73863e9e7b9587486b122e6aae, ASSIGN because future has completed 2024-11-24T22:14:21,615 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=32, ppid=31, state=RUNNABLE, hasLock=false; OpenRegionProcedure 63842f73863e9e7b9587486b122e6aae, server=bd6592c37bb2,40087,1732486423713}] 2024-11-24T22:14:21,771 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37701 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=30 2024-11-24T22:14:21,771 INFO [RS_OPEN_REGION-regionserver/bd6592c37bb2:0-0 {event_type=M_RS_OPEN_REGION, pid=32}] handler.AssignRegionHandler(132): Open TestAcidGuarantees,,1732486460633.63842f73863e9e7b9587486b122e6aae. 2024-11-24T22:14:21,771 DEBUG [RS_OPEN_REGION-regionserver/bd6592c37bb2:0-0 {event_type=M_RS_OPEN_REGION, pid=32}] regionserver.HRegion(7752): Opening region: {ENCODED => 63842f73863e9e7b9587486b122e6aae, NAME => 'TestAcidGuarantees,,1732486460633.63842f73863e9e7b9587486b122e6aae.', STARTKEY => '', ENDKEY => ''} 2024-11-24T22:14:21,772 DEBUG [RS_OPEN_REGION-regionserver/bd6592c37bb2:0-0 {event_type=M_RS_OPEN_REGION, pid=32}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table TestAcidGuarantees 63842f73863e9e7b9587486b122e6aae 2024-11-24T22:14:21,772 DEBUG [RS_OPEN_REGION-regionserver/bd6592c37bb2:0-0 {event_type=M_RS_OPEN_REGION, pid=32}] regionserver.HRegion(898): Instantiated TestAcidGuarantees,,1732486460633.63842f73863e9e7b9587486b122e6aae.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-24T22:14:21,772 DEBUG [RS_OPEN_REGION-regionserver/bd6592c37bb2:0-0 {event_type=M_RS_OPEN_REGION, pid=32}] regionserver.HRegion(7794): checking encryption for 63842f73863e9e7b9587486b122e6aae 2024-11-24T22:14:21,772 DEBUG [RS_OPEN_REGION-regionserver/bd6592c37bb2:0-0 {event_type=M_RS_OPEN_REGION, pid=32}] regionserver.HRegion(7797): checking classloading for 63842f73863e9e7b9587486b122e6aae 2024-11-24T22:14:21,774 INFO [StoreOpener-63842f73863e9e7b9587486b122e6aae-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family A of region 63842f73863e9e7b9587486b122e6aae 2024-11-24T22:14:21,775 INFO [StoreOpener-63842f73863e9e7b9587486b122e6aae-1 {}] regionserver.CompactingMemStore(122): Store=A, in-memory flush size threshold=38.40 KB, immutable segments index type=ARRAY_MAP, compactor=EAGER, pipelineThreshold=2, compactionCellMax=10 2024-11-24T22:14:21,776 INFO [StoreOpener-63842f73863e9e7b9587486b122e6aae-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 63842f73863e9e7b9587486b122e6aae columnFamilyName A 2024-11-24T22:14:21,776 DEBUG [StoreOpener-63842f73863e9e7b9587486b122e6aae-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T22:14:21,777 INFO [StoreOpener-63842f73863e9e7b9587486b122e6aae-1 {}] regionserver.HStore(327): Store=63842f73863e9e7b9587486b122e6aae/A, memstore type=CompactingMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-24T22:14:21,777 INFO [StoreOpener-63842f73863e9e7b9587486b122e6aae-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family B of region 63842f73863e9e7b9587486b122e6aae 2024-11-24T22:14:21,778 INFO [StoreOpener-63842f73863e9e7b9587486b122e6aae-1 {}] regionserver.CompactingMemStore(122): Store=B, in-memory flush size threshold=38.40 KB, immutable segments index type=ARRAY_MAP, compactor=EAGER, pipelineThreshold=2, compactionCellMax=10 2024-11-24T22:14:21,778 INFO [StoreOpener-63842f73863e9e7b9587486b122e6aae-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 63842f73863e9e7b9587486b122e6aae columnFamilyName B 2024-11-24T22:14:21,778 DEBUG [StoreOpener-63842f73863e9e7b9587486b122e6aae-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T22:14:21,779 INFO [StoreOpener-63842f73863e9e7b9587486b122e6aae-1 {}] regionserver.HStore(327): Store=63842f73863e9e7b9587486b122e6aae/B, memstore type=CompactingMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-24T22:14:21,779 INFO [StoreOpener-63842f73863e9e7b9587486b122e6aae-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family C of region 63842f73863e9e7b9587486b122e6aae 2024-11-24T22:14:21,780 INFO [StoreOpener-63842f73863e9e7b9587486b122e6aae-1 {}] regionserver.CompactingMemStore(122): Store=C, in-memory flush size threshold=38.40 KB, immutable segments index type=ARRAY_MAP, compactor=EAGER, pipelineThreshold=2, compactionCellMax=10 2024-11-24T22:14:21,780 INFO [StoreOpener-63842f73863e9e7b9587486b122e6aae-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 63842f73863e9e7b9587486b122e6aae columnFamilyName C 2024-11-24T22:14:21,780 DEBUG [StoreOpener-63842f73863e9e7b9587486b122e6aae-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T22:14:21,780 INFO [StoreOpener-63842f73863e9e7b9587486b122e6aae-1 {}] regionserver.HStore(327): Store=63842f73863e9e7b9587486b122e6aae/C, memstore type=CompactingMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-24T22:14:21,781 INFO [RS_OPEN_REGION-regionserver/bd6592c37bb2:0-0 {event_type=M_RS_OPEN_REGION, pid=32}] regionserver.HRegion(1212): Setting FlushNonSloppyStoresFirstPolicy for the region=TestAcidGuarantees,,1732486460633.63842f73863e9e7b9587486b122e6aae. 2024-11-24T22:14:21,781 DEBUG [RS_OPEN_REGION-regionserver/bd6592c37bb2:0-0 {event_type=M_RS_OPEN_REGION, pid=32}] regionserver.HRegion(1038): replaying wal for 63842f73863e9e7b9587486b122e6aae 2024-11-24T22:14:21,781 DEBUG [RS_OPEN_REGION-regionserver/bd6592c37bb2:0-0 {event_type=M_RS_OPEN_REGION, pid=32}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/63842f73863e9e7b9587486b122e6aae 2024-11-24T22:14:21,782 DEBUG [RS_OPEN_REGION-regionserver/bd6592c37bb2:0-0 {event_type=M_RS_OPEN_REGION, pid=32}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/63842f73863e9e7b9587486b122e6aae 2024-11-24T22:14:21,783 DEBUG [RS_OPEN_REGION-regionserver/bd6592c37bb2:0-0 {event_type=M_RS_OPEN_REGION, pid=32}] regionserver.HRegion(1048): stopping wal replay for 63842f73863e9e7b9587486b122e6aae 2024-11-24T22:14:21,783 DEBUG [RS_OPEN_REGION-regionserver/bd6592c37bb2:0-0 {event_type=M_RS_OPEN_REGION, pid=32}] regionserver.HRegion(1060): Cleaning up temporary data for 63842f73863e9e7b9587486b122e6aae 2024-11-24T22:14:21,783 DEBUG [RS_OPEN_REGION-regionserver/bd6592c37bb2:0-0 {event_type=M_RS_OPEN_REGION, pid=32}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table TestAcidGuarantees descriptor;using region.getMemStoreFlushHeapSize/# of families (16.0 M)) instead. 2024-11-24T22:14:21,784 DEBUG [RS_OPEN_REGION-regionserver/bd6592c37bb2:0-0 {event_type=M_RS_OPEN_REGION, pid=32}] regionserver.HRegion(1093): writing seq id for 63842f73863e9e7b9587486b122e6aae 2024-11-24T22:14:21,787 DEBUG [RS_OPEN_REGION-regionserver/bd6592c37bb2:0-0 {event_type=M_RS_OPEN_REGION, pid=32}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/63842f73863e9e7b9587486b122e6aae/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-24T22:14:21,787 INFO [RS_OPEN_REGION-regionserver/bd6592c37bb2:0-0 {event_type=M_RS_OPEN_REGION, pid=32}] regionserver.HRegion(1114): Opened 63842f73863e9e7b9587486b122e6aae; next sequenceid=2; ConstantSizeRegionSplitPolicy{desiredMaxFileSize=73136351, jitterRate=0.08981655538082123}, FlushLargeStoresPolicy{flushSizeLowerBound=16777216} 2024-11-24T22:14:21,787 DEBUG [RS_OPEN_REGION-regionserver/bd6592c37bb2:0-0 {event_type=M_RS_OPEN_REGION, pid=32}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 63842f73863e9e7b9587486b122e6aae 2024-11-24T22:14:21,788 DEBUG [RS_OPEN_REGION-regionserver/bd6592c37bb2:0-0 {event_type=M_RS_OPEN_REGION, pid=32}] regionserver.HRegion(1006): Region open journal for 63842f73863e9e7b9587486b122e6aae: Running coprocessor pre-open hook at 1732486461772Writing region info on filesystem at 1732486461772Initializing all the Stores at 1732486461773 (+1 ms)Instantiating store for column family {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732486461773Instantiating store for column family {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732486461774 (+1 ms)Instantiating store for column family {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732486461774Cleaning up temporary data from old regions at 1732486461783 (+9 ms)Running coprocessor post-open hooks at 1732486461787 (+4 ms)Region opened successfully at 1732486461788 (+1 ms) 2024-11-24T22:14:21,789 INFO [RS_OPEN_REGION-regionserver/bd6592c37bb2:0-0 {event_type=M_RS_OPEN_REGION, pid=32}] regionserver.HRegionServer(2236): Post open deploy tasks for TestAcidGuarantees,,1732486460633.63842f73863e9e7b9587486b122e6aae., pid=32, masterSystemTime=1732486461767 2024-11-24T22:14:21,791 DEBUG [RS_OPEN_REGION-regionserver/bd6592c37bb2:0-0 {event_type=M_RS_OPEN_REGION, pid=32}] regionserver.HRegionServer(2266): Finished post open deploy task for TestAcidGuarantees,,1732486460633.63842f73863e9e7b9587486b122e6aae. 2024-11-24T22:14:21,791 INFO [RS_OPEN_REGION-regionserver/bd6592c37bb2:0-0 {event_type=M_RS_OPEN_REGION, pid=32}] handler.AssignRegionHandler(153): Opened TestAcidGuarantees,,1732486460633.63842f73863e9e7b9587486b122e6aae. 2024-11-24T22:14:21,791 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=31 updating hbase:meta row=63842f73863e9e7b9587486b122e6aae, regionState=OPEN, openSeqNum=2, regionLocation=bd6592c37bb2,40087,1732486423713 2024-11-24T22:14:21,793 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=32, ppid=31, state=RUNNABLE, hasLock=false; OpenRegionProcedure 63842f73863e9e7b9587486b122e6aae, server=bd6592c37bb2,40087,1732486423713 because future has completed 2024-11-24T22:14:21,796 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=32, resume processing ppid=31 2024-11-24T22:14:21,796 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=32, ppid=31, state=SUCCESS, hasLock=false; OpenRegionProcedure 63842f73863e9e7b9587486b122e6aae, server=bd6592c37bb2,40087,1732486423713 in 179 msec 2024-11-24T22:14:21,798 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=31, resume processing ppid=30 2024-11-24T22:14:21,798 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=31, ppid=30, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=63842f73863e9e7b9587486b122e6aae, ASSIGN in 336 msec 2024-11-24T22:14:21,799 INFO [PEWorker-5 {}] procedure.CreateTableProcedure(89): pid=30, state=RUNNABLE:CREATE_TABLE_UPDATE_DESC_CACHE, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_UPDATE_DESC_CACHE 2024-11-24T22:14:21,799 DEBUG [PEWorker-5 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1732486461799"}]},"ts":"1732486461799"} 2024-11-24T22:14:21,801 INFO [PEWorker-5 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=ENABLED in hbase:meta 2024-11-24T22:14:21,802 INFO [PEWorker-5 {}] procedure.CreateTableProcedure(89): pid=30, state=RUNNABLE:CREATE_TABLE_POST_OPERATION, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_POST_OPERATION 2024-11-24T22:14:21,804 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=30, state=SUCCESS, hasLock=false; CreateTableProcedure table=TestAcidGuarantees in 1.1680 sec 2024-11-24T22:14:22,783 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37701 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=30 2024-11-24T22:14:22,783 INFO [RPCClient-NioEventLoopGroup-4-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: CREATE, Table Name: default:TestAcidGuarantees completed 2024-11-24T22:14:22,784 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@6ffee9aa, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-24T22:14:22,785 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request bd6592c37bb2,37701,-1 for getting cluster id 2024-11-24T22:14:22,785 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-24T22:14:22,787 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '3c461ac1-06fb-4739-8702-6368e685b03c' 2024-11-24T22:14:22,787 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-24T22:14:22,787 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "3c461ac1-06fb-4739-8702-6368e685b03c" 2024-11-24T22:14:22,787 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@73af3d7a, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-24T22:14:22,787 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [bd6592c37bb2,37701,-1] 2024-11-24T22:14:22,788 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-24T22:14:22,788 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-24T22:14:22,789 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:39232, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-24T22:14:22,790 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@366f7c77, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-24T22:14:22,790 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-24T22:14:22,791 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=bd6592c37bb2,40087,1732486423713, seqNum=-1] 2024-11-24T22:14:22,791 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-24T22:14:22,792 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:40176, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-24T22:14:22,794 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching master stub from registry 2024-11-24T22:14:22,795 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncConnectionImpl(321): The fetched master address is bd6592c37bb2,37701,1732486422992 2024-11-24T22:14:22,795 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.ConnectionUtils(555): The fetched master stub is org.apache.hadoop.hbase.shaded.protobuf.generated.MasterProtos$MasterService$Stub@678171c4 2024-11-24T22:14:22,795 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=MasterService, sasl=false 2024-11-24T22:14:22,796 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:39238, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=MasterService 2024-11-24T22:14:22,801 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37701 {}] util.TableDescriptorChecker(321): MEMSTORE_FLUSHSIZE for table descriptor or "hbase.hregion.memstore.flush.size" (131072) is too small, which might cause very frequent flushing. 2024-11-24T22:14:22,802 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37701 {}] master.HMaster$14(2863): Client=jenkins//172.17.0.2 modify table TestAcidGuarantees from 'TestAcidGuarantees', {TABLE_ATTRIBUTES => {METADATA => {'hbase.hregion.compacting.memstore.type' => 'EAGER', 'hbase.hregion.memstore.mslab.enabled' => 'false', 'hbase.memstore.inmemoryflush.threshold.factor' => '0.9', 'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} to 'TestAcidGuarantees', {TABLE_ATTRIBUTES => {METADATA => {'hbase.hregion.compacting.memstore.type' => 'EAGER', 'hbase.hregion.memstore.mslab.enabled' => 'false', 'hbase.memstore.inmemoryflush.threshold.factor' => '0.9', 'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', MOB_THRESHOLD => '4', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', IS_MOB => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-11-24T22:14:22,810 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37701 {}] procedure2.ProcedureExecutor(1139): Stored pid=33, state=RUNNABLE:MODIFY_TABLE_PREPARE, hasLock=false; ModifyTableProcedure table=TestAcidGuarantees 2024-11-24T22:14:22,833 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45929 is added to blk_1073741938_1114 (size=1097) 2024-11-24T22:14:23,237 DEBUG [PEWorker-4 {}] util.FSTableDescriptors(591): Deleted hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/.tabledesc/.tableinfo.0000000001.1061 2024-11-24T22:14:23,237 INFO [PEWorker-4 {}] util.FSTableDescriptors(361): Updated tableinfo=hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/.tabledesc/.tableinfo.0000000002.1097 2024-11-24T22:14:23,242 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=34, ppid=33, state=RUNNABLE:REOPEN_TABLE_REGIONS_GET_REGIONS, hasLock=false; ReopenTableRegionsProcedure table=TestAcidGuarantees}] 2024-11-24T22:14:23,250 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=35, ppid=34, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=63842f73863e9e7b9587486b122e6aae, REOPEN/MOVE}] 2024-11-24T22:14:23,251 INFO [PEWorker-2 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=35, ppid=34, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=63842f73863e9e7b9587486b122e6aae, REOPEN/MOVE 2024-11-24T22:14:23,252 INFO [PEWorker-2 {}] assignment.RegionStateStore(223): pid=35 updating hbase:meta row=63842f73863e9e7b9587486b122e6aae, regionState=CLOSING, regionLocation=bd6592c37bb2,40087,1732486423713 2024-11-24T22:14:23,254 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=35, ppid=34, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=63842f73863e9e7b9587486b122e6aae, REOPEN/MOVE because future has completed 2024-11-24T22:14:23,254 DEBUG [PEWorker-3 {}] assignment.TransitRegionStateProcedure(375): Close region: isSplit: false: evictOnSplit: true: evictOnClose: false 2024-11-24T22:14:23,254 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=36, ppid=35, state=RUNNABLE, hasLock=false; CloseRegionProcedure 63842f73863e9e7b9587486b122e6aae, server=bd6592c37bb2,40087,1732486423713}] 2024-11-24T22:14:23,407 INFO [RS_CLOSE_REGION-regionserver/bd6592c37bb2:0-0 {event_type=M_RS_CLOSE_REGION, pid=36}] handler.UnassignRegionHandler(122): Close 63842f73863e9e7b9587486b122e6aae 2024-11-24T22:14:23,407 DEBUG [RS_CLOSE_REGION-regionserver/bd6592c37bb2:0-0 {event_type=M_RS_CLOSE_REGION, pid=36}] handler.UnassignRegionHandler(136): Unassign region: split region: false: evictCache: false 2024-11-24T22:14:23,407 DEBUG [RS_CLOSE_REGION-regionserver/bd6592c37bb2:0-0 {event_type=M_RS_CLOSE_REGION, pid=36}] regionserver.HRegion(1722): Closing 63842f73863e9e7b9587486b122e6aae, disabling compactions & flushes 2024-11-24T22:14:23,407 INFO [RS_CLOSE_REGION-regionserver/bd6592c37bb2:0-0 {event_type=M_RS_CLOSE_REGION, pid=36}] regionserver.HRegion(1755): Closing region TestAcidGuarantees,,1732486460633.63842f73863e9e7b9587486b122e6aae. 2024-11-24T22:14:23,407 DEBUG [RS_CLOSE_REGION-regionserver/bd6592c37bb2:0-0 {event_type=M_RS_CLOSE_REGION, pid=36}] regionserver.HRegion(1776): Time limited wait for close lock on TestAcidGuarantees,,1732486460633.63842f73863e9e7b9587486b122e6aae. 2024-11-24T22:14:23,408 DEBUG [RS_CLOSE_REGION-regionserver/bd6592c37bb2:0-0 {event_type=M_RS_CLOSE_REGION, pid=36}] regionserver.HRegion(1843): Acquired close lock on TestAcidGuarantees,,1732486460633.63842f73863e9e7b9587486b122e6aae. after waiting 0 ms 2024-11-24T22:14:23,408 DEBUG [RS_CLOSE_REGION-regionserver/bd6592c37bb2:0-0 {event_type=M_RS_CLOSE_REGION, pid=36}] regionserver.HRegion(1853): Updates disabled for region TestAcidGuarantees,,1732486460633.63842f73863e9e7b9587486b122e6aae. 2024-11-24T22:14:23,412 DEBUG [RS_CLOSE_REGION-regionserver/bd6592c37bb2:0-0 {event_type=M_RS_CLOSE_REGION, pid=36}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/63842f73863e9e7b9587486b122e6aae/recovered.edits/4.seqid, newMaxSeqId=4, maxSeqId=1 2024-11-24T22:14:23,413 INFO [RS_CLOSE_REGION-regionserver/bd6592c37bb2:0-0 {event_type=M_RS_CLOSE_REGION, pid=36}] regionserver.HRegion(1973): Closed TestAcidGuarantees,,1732486460633.63842f73863e9e7b9587486b122e6aae. 2024-11-24T22:14:23,413 DEBUG [RS_CLOSE_REGION-regionserver/bd6592c37bb2:0-0 {event_type=M_RS_CLOSE_REGION, pid=36}] regionserver.HRegion(1676): Region close journal for 63842f73863e9e7b9587486b122e6aae: Waiting for close lock at 1732486463407Running coprocessor pre-close hooks at 1732486463407Disabling compacts and flushes for region at 1732486463407Disabling writes for close at 1732486463408 (+1 ms)Writing region close event to WAL at 1732486463408Running coprocessor post-close hooks at 1732486463413 (+5 ms)Closed at 1732486463413 2024-11-24T22:14:23,413 WARN [RS_CLOSE_REGION-regionserver/bd6592c37bb2:0-0 {event_type=M_RS_CLOSE_REGION, pid=36}] regionserver.HRegionServer(3299): Not adding moved region record: 63842f73863e9e7b9587486b122e6aae to self. 2024-11-24T22:14:23,415 INFO [RS_CLOSE_REGION-regionserver/bd6592c37bb2:0-0 {event_type=M_RS_CLOSE_REGION, pid=36}] handler.UnassignRegionHandler(157): Closed 63842f73863e9e7b9587486b122e6aae 2024-11-24T22:14:23,416 INFO [PEWorker-4 {}] assignment.RegionStateStore(223): pid=35 updating hbase:meta row=63842f73863e9e7b9587486b122e6aae, regionState=CLOSED 2024-11-24T22:14:23,417 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=36, ppid=35, state=RUNNABLE, hasLock=false; CloseRegionProcedure 63842f73863e9e7b9587486b122e6aae, server=bd6592c37bb2,40087,1732486423713 because future has completed 2024-11-24T22:14:23,420 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=36, resume processing ppid=35 2024-11-24T22:14:23,420 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=36, ppid=35, state=SUCCESS, hasLock=false; CloseRegionProcedure 63842f73863e9e7b9587486b122e6aae, server=bd6592c37bb2,40087,1732486423713 in 164 msec 2024-11-24T22:14:23,420 INFO [PEWorker-2 {}] assignment.TransitRegionStateProcedure(269): Starting pid=35, ppid=34, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=63842f73863e9e7b9587486b122e6aae, REOPEN/MOVE; state=CLOSED, location=bd6592c37bb2,40087,1732486423713; forceNewPlan=false, retain=true 2024-11-24T22:14:23,571 INFO [PEWorker-3 {}] assignment.RegionStateStore(223): pid=35 updating hbase:meta row=63842f73863e9e7b9587486b122e6aae, regionState=OPENING, regionLocation=bd6592c37bb2,40087,1732486423713 2024-11-24T22:14:23,573 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=35, ppid=34, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=63842f73863e9e7b9587486b122e6aae, REOPEN/MOVE because future has completed 2024-11-24T22:14:23,573 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=37, ppid=35, state=RUNNABLE, hasLock=false; OpenRegionProcedure 63842f73863e9e7b9587486b122e6aae, server=bd6592c37bb2,40087,1732486423713}] 2024-11-24T22:14:23,729 INFO [RS_OPEN_REGION-regionserver/bd6592c37bb2:0-0 {event_type=M_RS_OPEN_REGION, pid=37}] handler.AssignRegionHandler(132): Open TestAcidGuarantees,,1732486460633.63842f73863e9e7b9587486b122e6aae. 2024-11-24T22:14:23,729 DEBUG [RS_OPEN_REGION-regionserver/bd6592c37bb2:0-0 {event_type=M_RS_OPEN_REGION, pid=37}] regionserver.HRegion(7752): Opening region: {ENCODED => 63842f73863e9e7b9587486b122e6aae, NAME => 'TestAcidGuarantees,,1732486460633.63842f73863e9e7b9587486b122e6aae.', STARTKEY => '', ENDKEY => ''} 2024-11-24T22:14:23,730 DEBUG [RS_OPEN_REGION-regionserver/bd6592c37bb2:0-0 {event_type=M_RS_OPEN_REGION, pid=37}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table TestAcidGuarantees 63842f73863e9e7b9587486b122e6aae 2024-11-24T22:14:23,730 DEBUG [RS_OPEN_REGION-regionserver/bd6592c37bb2:0-0 {event_type=M_RS_OPEN_REGION, pid=37}] regionserver.HRegion(898): Instantiated TestAcidGuarantees,,1732486460633.63842f73863e9e7b9587486b122e6aae.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-24T22:14:23,730 DEBUG [RS_OPEN_REGION-regionserver/bd6592c37bb2:0-0 {event_type=M_RS_OPEN_REGION, pid=37}] regionserver.HRegion(7794): checking encryption for 63842f73863e9e7b9587486b122e6aae 2024-11-24T22:14:23,730 DEBUG [RS_OPEN_REGION-regionserver/bd6592c37bb2:0-0 {event_type=M_RS_OPEN_REGION, pid=37}] regionserver.HRegion(7797): checking classloading for 63842f73863e9e7b9587486b122e6aae 2024-11-24T22:14:23,733 INFO [StoreOpener-63842f73863e9e7b9587486b122e6aae-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family A of region 63842f73863e9e7b9587486b122e6aae 2024-11-24T22:14:23,734 INFO [StoreOpener-63842f73863e9e7b9587486b122e6aae-1 {}] regionserver.CompactingMemStore(122): Store=A, in-memory flush size threshold=38.40 KB, immutable segments index type=ARRAY_MAP, compactor=EAGER, pipelineThreshold=2, compactionCellMax=10 2024-11-24T22:14:23,739 INFO [StoreOpener-63842f73863e9e7b9587486b122e6aae-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 63842f73863e9e7b9587486b122e6aae columnFamilyName A 2024-11-24T22:14:23,741 DEBUG [StoreOpener-63842f73863e9e7b9587486b122e6aae-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T22:14:23,742 INFO [StoreOpener-63842f73863e9e7b9587486b122e6aae-1 {}] regionserver.HStore(327): Store=63842f73863e9e7b9587486b122e6aae/A, memstore type=CompactingMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-24T22:14:23,743 INFO [StoreOpener-63842f73863e9e7b9587486b122e6aae-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family B of region 63842f73863e9e7b9587486b122e6aae 2024-11-24T22:14:23,743 INFO [StoreOpener-63842f73863e9e7b9587486b122e6aae-1 {}] regionserver.CompactingMemStore(122): Store=B, in-memory flush size threshold=38.40 KB, immutable segments index type=ARRAY_MAP, compactor=EAGER, pipelineThreshold=2, compactionCellMax=10 2024-11-24T22:14:23,744 INFO [StoreOpener-63842f73863e9e7b9587486b122e6aae-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 63842f73863e9e7b9587486b122e6aae columnFamilyName B 2024-11-24T22:14:23,744 DEBUG [StoreOpener-63842f73863e9e7b9587486b122e6aae-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T22:14:23,745 INFO [StoreOpener-63842f73863e9e7b9587486b122e6aae-1 {}] regionserver.HStore(327): Store=63842f73863e9e7b9587486b122e6aae/B, memstore type=CompactingMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-24T22:14:23,745 INFO [StoreOpener-63842f73863e9e7b9587486b122e6aae-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family C of region 63842f73863e9e7b9587486b122e6aae 2024-11-24T22:14:23,746 INFO [StoreOpener-63842f73863e9e7b9587486b122e6aae-1 {}] regionserver.CompactingMemStore(122): Store=C, in-memory flush size threshold=38.40 KB, immutable segments index type=ARRAY_MAP, compactor=EAGER, pipelineThreshold=2, compactionCellMax=10 2024-11-24T22:14:23,746 INFO [StoreOpener-63842f73863e9e7b9587486b122e6aae-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 63842f73863e9e7b9587486b122e6aae columnFamilyName C 2024-11-24T22:14:23,746 DEBUG [StoreOpener-63842f73863e9e7b9587486b122e6aae-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T22:14:23,746 INFO [StoreOpener-63842f73863e9e7b9587486b122e6aae-1 {}] regionserver.HStore(327): Store=63842f73863e9e7b9587486b122e6aae/C, memstore type=CompactingMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-24T22:14:23,746 INFO [RS_OPEN_REGION-regionserver/bd6592c37bb2:0-0 {event_type=M_RS_OPEN_REGION, pid=37}] regionserver.HRegion(1212): Setting FlushNonSloppyStoresFirstPolicy for the region=TestAcidGuarantees,,1732486460633.63842f73863e9e7b9587486b122e6aae. 2024-11-24T22:14:23,747 DEBUG [RS_OPEN_REGION-regionserver/bd6592c37bb2:0-0 {event_type=M_RS_OPEN_REGION, pid=37}] regionserver.HRegion(1038): replaying wal for 63842f73863e9e7b9587486b122e6aae 2024-11-24T22:14:23,747 DEBUG [RS_OPEN_REGION-regionserver/bd6592c37bb2:0-0 {event_type=M_RS_OPEN_REGION, pid=37}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/63842f73863e9e7b9587486b122e6aae 2024-11-24T22:14:23,748 DEBUG [RS_OPEN_REGION-regionserver/bd6592c37bb2:0-0 {event_type=M_RS_OPEN_REGION, pid=37}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/63842f73863e9e7b9587486b122e6aae 2024-11-24T22:14:23,749 DEBUG [RS_OPEN_REGION-regionserver/bd6592c37bb2:0-0 {event_type=M_RS_OPEN_REGION, pid=37}] regionserver.HRegion(1048): stopping wal replay for 63842f73863e9e7b9587486b122e6aae 2024-11-24T22:14:23,749 DEBUG [RS_OPEN_REGION-regionserver/bd6592c37bb2:0-0 {event_type=M_RS_OPEN_REGION, pid=37}] regionserver.HRegion(1060): Cleaning up temporary data for 63842f73863e9e7b9587486b122e6aae 2024-11-24T22:14:23,750 DEBUG [RS_OPEN_REGION-regionserver/bd6592c37bb2:0-0 {event_type=M_RS_OPEN_REGION, pid=37}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table TestAcidGuarantees descriptor;using region.getMemStoreFlushHeapSize/# of families (16.0 M)) instead. 2024-11-24T22:14:23,751 DEBUG [RS_OPEN_REGION-regionserver/bd6592c37bb2:0-0 {event_type=M_RS_OPEN_REGION, pid=37}] regionserver.HRegion(1093): writing seq id for 63842f73863e9e7b9587486b122e6aae 2024-11-24T22:14:23,752 INFO [RS_OPEN_REGION-regionserver/bd6592c37bb2:0-0 {event_type=M_RS_OPEN_REGION, pid=37}] regionserver.HRegion(1114): Opened 63842f73863e9e7b9587486b122e6aae; next sequenceid=5; ConstantSizeRegionSplitPolicy{desiredMaxFileSize=68920062, jitterRate=0.026988953351974487}, FlushLargeStoresPolicy{flushSizeLowerBound=16777216} 2024-11-24T22:14:23,752 DEBUG [RS_OPEN_REGION-regionserver/bd6592c37bb2:0-0 {event_type=M_RS_OPEN_REGION, pid=37}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 63842f73863e9e7b9587486b122e6aae 2024-11-24T22:14:23,754 DEBUG [RS_OPEN_REGION-regionserver/bd6592c37bb2:0-0 {event_type=M_RS_OPEN_REGION, pid=37}] regionserver.HRegion(1006): Region open journal for 63842f73863e9e7b9587486b122e6aae: Running coprocessor pre-open hook at 1732486463730Writing region info on filesystem at 1732486463730Initializing all the Stores at 1732486463731 (+1 ms)Instantiating store for column family {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', MOB_THRESHOLD => '4', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', IS_MOB => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732486463731Instantiating store for column family {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732486463732 (+1 ms)Instantiating store for column family {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732486463732Cleaning up temporary data from old regions at 1732486463749 (+17 ms)Running coprocessor post-open hooks at 1732486463752 (+3 ms)Region opened successfully at 1732486463754 (+2 ms) 2024-11-24T22:14:23,755 INFO [RS_OPEN_REGION-regionserver/bd6592c37bb2:0-0 {event_type=M_RS_OPEN_REGION, pid=37}] regionserver.HRegionServer(2236): Post open deploy tasks for TestAcidGuarantees,,1732486460633.63842f73863e9e7b9587486b122e6aae., pid=37, masterSystemTime=1732486463725 2024-11-24T22:14:23,757 DEBUG [RS_OPEN_REGION-regionserver/bd6592c37bb2:0-0 {event_type=M_RS_OPEN_REGION, pid=37}] regionserver.HRegionServer(2266): Finished post open deploy task for TestAcidGuarantees,,1732486460633.63842f73863e9e7b9587486b122e6aae. 2024-11-24T22:14:23,757 INFO [RS_OPEN_REGION-regionserver/bd6592c37bb2:0-0 {event_type=M_RS_OPEN_REGION, pid=37}] handler.AssignRegionHandler(153): Opened TestAcidGuarantees,,1732486460633.63842f73863e9e7b9587486b122e6aae. 2024-11-24T22:14:23,758 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=35 updating hbase:meta row=63842f73863e9e7b9587486b122e6aae, regionState=OPEN, openSeqNum=5, regionLocation=bd6592c37bb2,40087,1732486423713 2024-11-24T22:14:23,760 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=37, ppid=35, state=RUNNABLE, hasLock=false; OpenRegionProcedure 63842f73863e9e7b9587486b122e6aae, server=bd6592c37bb2,40087,1732486423713 because future has completed 2024-11-24T22:14:23,764 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=37, resume processing ppid=35 2024-11-24T22:14:23,764 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=37, ppid=35, state=SUCCESS, hasLock=false; OpenRegionProcedure 63842f73863e9e7b9587486b122e6aae, server=bd6592c37bb2,40087,1732486423713 in 188 msec 2024-11-24T22:14:23,766 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=35, resume processing ppid=34 2024-11-24T22:14:23,766 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=35, ppid=34, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=63842f73863e9e7b9587486b122e6aae, REOPEN/MOVE in 514 msec 2024-11-24T22:14:23,771 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=34, resume processing ppid=33 2024-11-24T22:14:23,771 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=34, ppid=33, state=SUCCESS, hasLock=false; ReopenTableRegionsProcedure table=TestAcidGuarantees in 527 msec 2024-11-24T22:14:23,775 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=33, state=SUCCESS, hasLock=false; ModifyTableProcedure table=TestAcidGuarantees in 968 msec 2024-11-24T22:14:23,777 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37701 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=33 2024-11-24T22:14:23,777 INFO [RPCClient-NioEventLoopGroup-4-7 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: MODIFY_COLUMN_FAMILY, Table Name: default:TestAcidGuarantees completed 2024-11-24T22:14:23,778 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@2f683619, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-24T22:14:23,778 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request bd6592c37bb2,37701,-1 for getting cluster id 2024-11-24T22:14:23,778 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-24T22:14:23,779 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '3c461ac1-06fb-4739-8702-6368e685b03c' 2024-11-24T22:14:23,779 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-24T22:14:23,780 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "3c461ac1-06fb-4739-8702-6368e685b03c" 2024-11-24T22:14:23,780 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@511d5f51, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-24T22:14:23,780 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [bd6592c37bb2,37701,-1] 2024-11-24T22:14:23,780 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-24T22:14:23,780 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-24T22:14:23,782 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:39256, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-24T22:14:23,783 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@1f5c30e4, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-24T22:14:23,783 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@dd47c94, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-24T22:14:23,783 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request bd6592c37bb2,37701,-1 for getting cluster id 2024-11-24T22:14:23,783 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-24T22:14:23,784 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '3c461ac1-06fb-4739-8702-6368e685b03c' 2024-11-24T22:14:23,784 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-24T22:14:23,784 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "3c461ac1-06fb-4739-8702-6368e685b03c" 2024-11-24T22:14:23,785 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@7d54e90b, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-24T22:14:23,785 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [bd6592c37bb2,37701,-1] 2024-11-24T22:14:23,785 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-24T22:14:23,785 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-24T22:14:23,786 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:39280, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-24T22:14:23,787 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@56eb7834, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-24T22:14:23,788 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@750c1845, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-24T22:14:23,788 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request bd6592c37bb2,37701,-1 for getting cluster id 2024-11-24T22:14:23,788 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-24T22:14:23,788 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '3c461ac1-06fb-4739-8702-6368e685b03c' 2024-11-24T22:14:23,788 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-24T22:14:23,789 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "3c461ac1-06fb-4739-8702-6368e685b03c" 2024-11-24T22:14:23,789 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@3615b36c, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-24T22:14:23,789 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [bd6592c37bb2,37701,-1] 2024-11-24T22:14:23,789 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-24T22:14:23,789 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-24T22:14:23,790 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:39286, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-24T22:14:23,790 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@4c024fe, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-24T22:14:23,791 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@85a5f7a, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-24T22:14:23,791 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request bd6592c37bb2,37701,-1 for getting cluster id 2024-11-24T22:14:23,791 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-24T22:14:23,792 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '3c461ac1-06fb-4739-8702-6368e685b03c' 2024-11-24T22:14:23,792 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-24T22:14:23,793 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "3c461ac1-06fb-4739-8702-6368e685b03c" 2024-11-24T22:14:23,793 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@20b7a44f, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-24T22:14:23,793 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [bd6592c37bb2,37701,-1] 2024-11-24T22:14:23,793 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-24T22:14:23,793 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-24T22:14:23,794 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:39310, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-24T22:14:23,794 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@efeba7b, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-24T22:14:23,795 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@71151d6d, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-24T22:14:23,795 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request bd6592c37bb2,37701,-1 for getting cluster id 2024-11-24T22:14:23,795 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-24T22:14:23,796 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '3c461ac1-06fb-4739-8702-6368e685b03c' 2024-11-24T22:14:23,796 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-24T22:14:23,796 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "3c461ac1-06fb-4739-8702-6368e685b03c" 2024-11-24T22:14:23,796 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@3c5f681e, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-24T22:14:23,797 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [bd6592c37bb2,37701,-1] 2024-11-24T22:14:23,797 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-24T22:14:23,797 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-24T22:14:23,798 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:39338, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-24T22:14:23,799 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@b7d7a2, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-24T22:14:23,799 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@6ab1fb6e, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-24T22:14:23,799 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request bd6592c37bb2,37701,-1 for getting cluster id 2024-11-24T22:14:23,800 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-24T22:14:23,800 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '3c461ac1-06fb-4739-8702-6368e685b03c' 2024-11-24T22:14:23,800 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-24T22:14:23,801 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "3c461ac1-06fb-4739-8702-6368e685b03c" 2024-11-24T22:14:23,801 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@2e4c492c, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-24T22:14:23,801 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [bd6592c37bb2,37701,-1] 2024-11-24T22:14:23,801 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-24T22:14:23,801 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-24T22:14:23,802 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:39362, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-24T22:14:23,802 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@21a435b1, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-24T22:14:23,803 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@6dd18334, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-24T22:14:23,803 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request bd6592c37bb2,37701,-1 for getting cluster id 2024-11-24T22:14:23,803 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-24T22:14:23,804 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '3c461ac1-06fb-4739-8702-6368e685b03c' 2024-11-24T22:14:23,804 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-24T22:14:23,804 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "3c461ac1-06fb-4739-8702-6368e685b03c" 2024-11-24T22:14:23,804 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@2fb22a63, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-24T22:14:23,804 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [bd6592c37bb2,37701,-1] 2024-11-24T22:14:23,804 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-24T22:14:23,804 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-24T22:14:23,805 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:39380, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-24T22:14:23,806 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@4c750091, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-24T22:14:23,806 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@52c57d01, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-24T22:14:23,806 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request bd6592c37bb2,37701,-1 for getting cluster id 2024-11-24T22:14:23,807 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-24T22:14:23,807 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '3c461ac1-06fb-4739-8702-6368e685b03c' 2024-11-24T22:14:23,807 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-24T22:14:23,807 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "3c461ac1-06fb-4739-8702-6368e685b03c" 2024-11-24T22:14:23,808 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@75cf965, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-24T22:14:23,808 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [bd6592c37bb2,37701,-1] 2024-11-24T22:14:23,808 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-24T22:14:23,808 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-24T22:14:23,809 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:39388, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-24T22:14:23,809 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@5dc3611c, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-24T22:14:23,810 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@4b465b92, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-24T22:14:23,810 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request bd6592c37bb2,37701,-1 for getting cluster id 2024-11-24T22:14:23,810 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-24T22:14:23,811 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '3c461ac1-06fb-4739-8702-6368e685b03c' 2024-11-24T22:14:23,811 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-24T22:14:23,811 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "3c461ac1-06fb-4739-8702-6368e685b03c" 2024-11-24T22:14:23,811 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@6baebe5d, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-24T22:14:23,811 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [bd6592c37bb2,37701,-1] 2024-11-24T22:14:23,811 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-24T22:14:23,811 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-24T22:14:23,812 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:39402, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-24T22:14:23,813 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@2ad43a4e, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-24T22:14:23,817 DEBUG [Thread-596 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-24T22:14:23,817 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37701 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-24T22:14:23,818 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=bd6592c37bb2,40087,1732486423713, seqNum=-1] 2024-11-24T22:14:23,818 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37701 {}] procedure2.ProcedureExecutor(1139): Stored pid=38, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=38, table=TestAcidGuarantees 2024-11-24T22:14:23,818 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-24T22:14:23,819 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37701 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=38 2024-11-24T22:14:23,819 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=38, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=38, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-24T22:14:23,819 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:40186, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-24T22:14:23,821 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=38, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=38, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-24T22:14:23,821 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=39, ppid=38, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-24T22:14:23,821 DEBUG [Thread-589 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-24T22:14:23,822 DEBUG [Thread-602 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-24T22:14:23,823 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=bd6592c37bb2,40087,1732486423713, seqNum=-1] 2024-11-24T22:14:23,823 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_0', locateType=CURRENT is [region=TestAcidGuarantees,,1732486460633.63842f73863e9e7b9587486b122e6aae., hostname=bd6592c37bb2,40087,1732486423713, seqNum=5] 2024-11-24T22:14:23,823 DEBUG [Thread-598 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-24T22:14:23,824 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-24T22:14:23,824 DEBUG [Thread-589 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=bd6592c37bb2,40087,1732486423713, seqNum=-1] 2024-11-24T22:14:23,824 DEBUG [Thread-587 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-24T22:14:23,825 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=bd6592c37bb2,40087,1732486423713, seqNum=-1] 2024-11-24T22:14:23,825 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:40196, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-24T22:14:23,825 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-24T22:14:23,826 DEBUG [Thread-589 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-24T22:14:23,826 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=bd6592c37bb2,40087,1732486423713, seqNum=-1] 2024-11-24T22:14:23,826 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:40200, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-24T22:14:23,826 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-24T22:14:23,827 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='', locateType=CURRENT is [region=TestAcidGuarantees,,1732486460633.63842f73863e9e7b9587486b122e6aae., hostname=bd6592c37bb2,40087,1732486423713, seqNum=5] 2024-11-24T22:14:23,827 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:40202, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-24T22:14:23,828 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_1', locateType=CURRENT is [region=TestAcidGuarantees,,1732486460633.63842f73863e9e7b9587486b122e6aae., hostname=bd6592c37bb2,40087,1732486423713, seqNum=5] 2024-11-24T22:14:23,828 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:40206, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-24T22:14:23,830 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_0', locateType=CURRENT is [region=TestAcidGuarantees,,1732486460633.63842f73863e9e7b9587486b122e6aae., hostname=bd6592c37bb2,40087,1732486423713, seqNum=5] 2024-11-24T22:14:23,831 DEBUG [Thread-589 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_1', locateType=CURRENT is [region=TestAcidGuarantees,,1732486460633.63842f73863e9e7b9587486b122e6aae., hostname=bd6592c37bb2,40087,1732486423713, seqNum=5] 2024-11-24T22:14:23,833 DEBUG [Thread-600 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-24T22:14:23,837 DEBUG [Thread-600 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=bd6592c37bb2,40087,1732486423713, seqNum=-1] 2024-11-24T22:14:23,837 DEBUG [Thread-600 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-24T22:14:23,839 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:40210, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-24T22:14:23,841 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='', locateType=CURRENT is [region=TestAcidGuarantees,,1732486460633.63842f73863e9e7b9587486b122e6aae., hostname=bd6592c37bb2,40087,1732486423713, seqNum=5] 2024-11-24T22:14:23,845 DEBUG [Thread-591 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-24T22:14:23,847 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=bd6592c37bb2,40087,1732486423713, seqNum=-1] 2024-11-24T22:14:23,847 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-24T22:14:23,848 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:40226, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-24T22:14:23,851 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_0', locateType=CURRENT is [region=TestAcidGuarantees,,1732486460633.63842f73863e9e7b9587486b122e6aae., hostname=bd6592c37bb2,40087,1732486423713, seqNum=5] 2024-11-24T22:14:23,853 DEBUG [Thread-593 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-24T22:14:23,854 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=bd6592c37bb2,40087,1732486423713, seqNum=-1] 2024-11-24T22:14:23,854 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-24T22:14:23,855 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:40232, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-24T22:14:23,858 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_0', locateType=CURRENT is [region=TestAcidGuarantees,,1732486460633.63842f73863e9e7b9587486b122e6aae., hostname=bd6592c37bb2,40087,1732486423713, seqNum=5] 2024-11-24T22:14:23,859 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 63842f73863e9e7b9587486b122e6aae 3/3 column families, dataSize=46.96 KB heapSize=140.20 KB 2024-11-24T22:14:23,859 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40087 {}] regionserver.HRegion(8855): Flush requested on 63842f73863e9e7b9587486b122e6aae 2024-11-24T22:14:23,861 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 63842f73863e9e7b9587486b122e6aae, store=A 2024-11-24T22:14:23,861 DEBUG [Thread-585 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-24T22:14:23,861 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-24T22:14:23,861 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 63842f73863e9e7b9587486b122e6aae, store=B 2024-11-24T22:14:23,861 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-24T22:14:23,861 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 63842f73863e9e7b9587486b122e6aae, store=C 2024-11-24T22:14:23,861 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-24T22:14:23,862 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=bd6592c37bb2,40087,1732486423713, seqNum=-1] 2024-11-24T22:14:23,862 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-24T22:14:23,864 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:40244, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-24T22:14:23,866 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_2', locateType=CURRENT is [region=TestAcidGuarantees,,1732486460633.63842f73863e9e7b9587486b122e6aae., hostname=bd6592c37bb2,40087,1732486423713, seqNum=5] 2024-11-24T22:14:23,914 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/bd6592c37bb2:0-6 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1732486463880, max timestamp=1732486463910Num uniques 859; 2024-11-24T22:14:23,915 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/bd6592c37bb2:0-6 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-11-24T22:14:23,916 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/bd6592c37bb2:0-9 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1732486463880, max timestamp=1732486463911Num uniques 859; 2024-11-24T22:14:23,916 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/bd6592c37bb2:0-9 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-11-24T22:14:23,919 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/bd6592c37bb2:0-8 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=150, cellSize=6870, totalHeapSize=15840, min timestamp=1732486463880, max timestamp=1732486463910Num uniques 859; 2024-11-24T22:14:23,919 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/bd6592c37bb2:0-8 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=6870, suffix heap size=117064,new segment heap size=15840 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=150 2024-11-24T22:14:23,921 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37701 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=38 2024-11-24T22:14:23,950 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411244231586579d04048854e6a51911a219c_63842f73863e9e7b9587486b122e6aae is 50, key is test_row_0/A:col10/1732486463857/Put/seqid=0 2024-11-24T22:14:23,971 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45929 is added to blk_1073741939_1115 (size=19474) 2024-11-24T22:14:23,978 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=40087 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=39 2024-11-24T22:14:23,982 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40087 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=63842f73863e9e7b9587486b122e6aae, server=bd6592c37bb2,40087,1732486423713 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-24T22:14:23,982 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bd6592c37bb2:0-2 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732486460633.63842f73863e9e7b9587486b122e6aae. 2024-11-24T22:14:23,982 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bd6592c37bb2:0-2 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732486460633.63842f73863e9e7b9587486b122e6aae. as already flushing 2024-11-24T22:14:23,982 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bd6592c37bb2:0-2 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732486460633.63842f73863e9e7b9587486b122e6aae. 2024-11-24T22:14:23,982 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40087 {}] ipc.CallRunner(138): callId: 8 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40226 deadline: 1732486523968, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=63842f73863e9e7b9587486b122e6aae, server=bd6592c37bb2,40087,1732486423713 2024-11-24T22:14:23,982 ERROR [RS_FLUSH_OPERATIONS-regionserver/bd6592c37bb2:0-2 {event_type=RS_FLUSH_REGIONS, pid=39}] handler.RSProcedureHandler(58): pid=39 java.io.IOException: Unable to complete flush {ENCODED => 63842f73863e9e7b9587486b122e6aae, NAME => 'TestAcidGuarantees,,1732486460633.63842f73863e9e7b9587486b122e6aae.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T22:14:23,982 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bd6592c37bb2:0-2 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=39 java.io.IOException: Unable to complete flush {ENCODED => 63842f73863e9e7b9587486b122e6aae, NAME => 'TestAcidGuarantees,,1732486460633.63842f73863e9e7b9587486b122e6aae.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T22:14:23,983 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732486460633.63842f73863e9e7b9587486b122e6aae., hostname=bd6592c37bb2,40087,1732486423713, seqNum=5 , the old value is region=TestAcidGuarantees,,1732486460633.63842f73863e9e7b9587486b122e6aae., hostname=bd6592c37bb2,40087,1732486423713, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=63842f73863e9e7b9587486b122e6aae, server=bd6592c37bb2,40087,1732486423713 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T22:14:23,983 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732486460633.63842f73863e9e7b9587486b122e6aae., hostname=bd6592c37bb2,40087,1732486423713, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=63842f73863e9e7b9587486b122e6aae, server=bd6592c37bb2,40087,1732486423713 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T22:14:23,983 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732486460633.63842f73863e9e7b9587486b122e6aae., hostname=bd6592c37bb2,40087,1732486423713, seqNum=5 because the exception is null or not the one we care about 2024-11-24T22:14:23,983 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37701 {}] master.HMaster(4177): Remote procedure failed, pid=39 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 63842f73863e9e7b9587486b122e6aae, NAME => 'TestAcidGuarantees,,1732486460633.63842f73863e9e7b9587486b122e6aae.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 63842f73863e9e7b9587486b122e6aae, NAME => 'TestAcidGuarantees,,1732486460633.63842f73863e9e7b9587486b122e6aae.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T22:14:23,983 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40087 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=63842f73863e9e7b9587486b122e6aae, server=bd6592c37bb2,40087,1732486423713 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-24T22:14:23,983 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40087 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40206 deadline: 1732486523969, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=63842f73863e9e7b9587486b122e6aae, server=bd6592c37bb2,40087,1732486423713 2024-11-24T22:14:23,984 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732486460633.63842f73863e9e7b9587486b122e6aae., hostname=bd6592c37bb2,40087,1732486423713, seqNum=5 , the old value is region=TestAcidGuarantees,,1732486460633.63842f73863e9e7b9587486b122e6aae., hostname=bd6592c37bb2,40087,1732486423713, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=63842f73863e9e7b9587486b122e6aae, server=bd6592c37bb2,40087,1732486423713 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T22:14:23,984 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732486460633.63842f73863e9e7b9587486b122e6aae., hostname=bd6592c37bb2,40087,1732486423713, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=63842f73863e9e7b9587486b122e6aae, server=bd6592c37bb2,40087,1732486423713 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T22:14:23,984 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732486460633.63842f73863e9e7b9587486b122e6aae., hostname=bd6592c37bb2,40087,1732486423713, seqNum=5 because the exception is null or not the one we care about 2024-11-24T22:14:23,990 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40087 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=63842f73863e9e7b9587486b122e6aae, server=bd6592c37bb2,40087,1732486423713 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-24T22:14:23,990 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40087 {}] ipc.CallRunner(138): callId: 7 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40244 deadline: 1732486523972, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=63842f73863e9e7b9587486b122e6aae, server=bd6592c37bb2,40087,1732486423713 2024-11-24T22:14:23,991 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732486460633.63842f73863e9e7b9587486b122e6aae., hostname=bd6592c37bb2,40087,1732486423713, seqNum=5 , the old value is region=TestAcidGuarantees,,1732486460633.63842f73863e9e7b9587486b122e6aae., hostname=bd6592c37bb2,40087,1732486423713, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=63842f73863e9e7b9587486b122e6aae, server=bd6592c37bb2,40087,1732486423713 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T22:14:23,991 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732486460633.63842f73863e9e7b9587486b122e6aae., hostname=bd6592c37bb2,40087,1732486423713, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=63842f73863e9e7b9587486b122e6aae, server=bd6592c37bb2,40087,1732486423713 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T22:14:23,991 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732486460633.63842f73863e9e7b9587486b122e6aae., hostname=bd6592c37bb2,40087,1732486423713, seqNum=5 because the exception is null or not the one we care about 2024-11-24T22:14:23,991 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40087 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=63842f73863e9e7b9587486b122e6aae, server=bd6592c37bb2,40087,1732486423713 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-24T22:14:23,991 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40087 {}] ipc.CallRunner(138): callId: 12 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40202 deadline: 1732486523974, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=63842f73863e9e7b9587486b122e6aae, server=bd6592c37bb2,40087,1732486423713 2024-11-24T22:14:23,992 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40087 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=63842f73863e9e7b9587486b122e6aae, server=bd6592c37bb2,40087,1732486423713 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-24T22:14:23,992 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732486460633.63842f73863e9e7b9587486b122e6aae., hostname=bd6592c37bb2,40087,1732486423713, seqNum=5 , the old value is region=TestAcidGuarantees,,1732486460633.63842f73863e9e7b9587486b122e6aae., hostname=bd6592c37bb2,40087,1732486423713, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=63842f73863e9e7b9587486b122e6aae, server=bd6592c37bb2,40087,1732486423713 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T22:14:23,992 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40087 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40232 deadline: 1732486523974, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=63842f73863e9e7b9587486b122e6aae, server=bd6592c37bb2,40087,1732486423713 2024-11-24T22:14:23,992 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732486460633.63842f73863e9e7b9587486b122e6aae., hostname=bd6592c37bb2,40087,1732486423713, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=63842f73863e9e7b9587486b122e6aae, server=bd6592c37bb2,40087,1732486423713 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T22:14:23,992 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732486460633.63842f73863e9e7b9587486b122e6aae., hostname=bd6592c37bb2,40087,1732486423713, seqNum=5 because the exception is null or not the one we care about 2024-11-24T22:14:23,992 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732486460633.63842f73863e9e7b9587486b122e6aae., hostname=bd6592c37bb2,40087,1732486423713, seqNum=5 , the old value is region=TestAcidGuarantees,,1732486460633.63842f73863e9e7b9587486b122e6aae., hostname=bd6592c37bb2,40087,1732486423713, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=63842f73863e9e7b9587486b122e6aae, server=bd6592c37bb2,40087,1732486423713 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T22:14:23,992 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732486460633.63842f73863e9e7b9587486b122e6aae., hostname=bd6592c37bb2,40087,1732486423713, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=63842f73863e9e7b9587486b122e6aae, server=bd6592c37bb2,40087,1732486423713 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T22:14:23,992 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732486460633.63842f73863e9e7b9587486b122e6aae., hostname=bd6592c37bb2,40087,1732486423713, seqNum=5 because the exception is null or not the one we care about 2024-11-24T22:14:24,098 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40087 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=63842f73863e9e7b9587486b122e6aae, server=bd6592c37bb2,40087,1732486423713 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-24T22:14:24,098 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40087 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40226 deadline: 1732486524092, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=63842f73863e9e7b9587486b122e6aae, server=bd6592c37bb2,40087,1732486423713 2024-11-24T22:14:24,099 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732486460633.63842f73863e9e7b9587486b122e6aae., hostname=bd6592c37bb2,40087,1732486423713, seqNum=5 , the old value is region=TestAcidGuarantees,,1732486460633.63842f73863e9e7b9587486b122e6aae., hostname=bd6592c37bb2,40087,1732486423713, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=63842f73863e9e7b9587486b122e6aae, server=bd6592c37bb2,40087,1732486423713 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T22:14:24,099 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732486460633.63842f73863e9e7b9587486b122e6aae., hostname=bd6592c37bb2,40087,1732486423713, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=63842f73863e9e7b9587486b122e6aae, server=bd6592c37bb2,40087,1732486423713 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T22:14:24,099 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732486460633.63842f73863e9e7b9587486b122e6aae., hostname=bd6592c37bb2,40087,1732486423713, seqNum=5 because the exception is null or not the one we care about 2024-11-24T22:14:24,099 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40087 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=63842f73863e9e7b9587486b122e6aae, server=bd6592c37bb2,40087,1732486423713 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-24T22:14:24,099 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40087 {}] ipc.CallRunner(138): callId: 11 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40206 deadline: 1732486524092, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=63842f73863e9e7b9587486b122e6aae, server=bd6592c37bb2,40087,1732486423713 2024-11-24T22:14:24,099 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732486460633.63842f73863e9e7b9587486b122e6aae., hostname=bd6592c37bb2,40087,1732486423713, seqNum=5 , the old value is region=TestAcidGuarantees,,1732486460633.63842f73863e9e7b9587486b122e6aae., hostname=bd6592c37bb2,40087,1732486423713, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=63842f73863e9e7b9587486b122e6aae, server=bd6592c37bb2,40087,1732486423713 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T22:14:24,100 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732486460633.63842f73863e9e7b9587486b122e6aae., hostname=bd6592c37bb2,40087,1732486423713, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=63842f73863e9e7b9587486b122e6aae, server=bd6592c37bb2,40087,1732486423713 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T22:14:24,100 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732486460633.63842f73863e9e7b9587486b122e6aae., hostname=bd6592c37bb2,40087,1732486423713, seqNum=5 because the exception is null or not the one we care about 2024-11-24T22:14:24,107 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40087 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=63842f73863e9e7b9587486b122e6aae, server=bd6592c37bb2,40087,1732486423713 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-24T22:14:24,107 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40087 {}] ipc.CallRunner(138): callId: 13 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40202 deadline: 1732486524101, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=63842f73863e9e7b9587486b122e6aae, server=bd6592c37bb2,40087,1732486423713 2024-11-24T22:14:24,107 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732486460633.63842f73863e9e7b9587486b122e6aae., hostname=bd6592c37bb2,40087,1732486423713, seqNum=5 , the old value is region=TestAcidGuarantees,,1732486460633.63842f73863e9e7b9587486b122e6aae., hostname=bd6592c37bb2,40087,1732486423713, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=63842f73863e9e7b9587486b122e6aae, server=bd6592c37bb2,40087,1732486423713 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T22:14:24,107 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732486460633.63842f73863e9e7b9587486b122e6aae., hostname=bd6592c37bb2,40087,1732486423713, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=63842f73863e9e7b9587486b122e6aae, server=bd6592c37bb2,40087,1732486423713 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T22:14:24,108 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732486460633.63842f73863e9e7b9587486b122e6aae., hostname=bd6592c37bb2,40087,1732486423713, seqNum=5 because the exception is null or not the one we care about 2024-11-24T22:14:24,108 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40087 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=63842f73863e9e7b9587486b122e6aae, server=bd6592c37bb2,40087,1732486423713 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-24T22:14:24,108 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40087 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40232 deadline: 1732486524102, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=63842f73863e9e7b9587486b122e6aae, server=bd6592c37bb2,40087,1732486423713 2024-11-24T22:14:24,108 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732486460633.63842f73863e9e7b9587486b122e6aae., hostname=bd6592c37bb2,40087,1732486423713, seqNum=5 , the old value is region=TestAcidGuarantees,,1732486460633.63842f73863e9e7b9587486b122e6aae., hostname=bd6592c37bb2,40087,1732486423713, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=63842f73863e9e7b9587486b122e6aae, server=bd6592c37bb2,40087,1732486423713 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T22:14:24,108 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732486460633.63842f73863e9e7b9587486b122e6aae., hostname=bd6592c37bb2,40087,1732486423713, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=63842f73863e9e7b9587486b122e6aae, server=bd6592c37bb2,40087,1732486423713 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T22:14:24,108 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732486460633.63842f73863e9e7b9587486b122e6aae., hostname=bd6592c37bb2,40087,1732486423713, seqNum=5 because the exception is null or not the one we care about 2024-11-24T22:14:24,109 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40087 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=63842f73863e9e7b9587486b122e6aae, server=bd6592c37bb2,40087,1732486423713 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-24T22:14:24,109 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40087 {}] ipc.CallRunner(138): callId: 8 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40244 deadline: 1732486524102, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=63842f73863e9e7b9587486b122e6aae, server=bd6592c37bb2,40087,1732486423713 2024-11-24T22:14:24,109 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732486460633.63842f73863e9e7b9587486b122e6aae., hostname=bd6592c37bb2,40087,1732486423713, seqNum=5 , the old value is region=TestAcidGuarantees,,1732486460633.63842f73863e9e7b9587486b122e6aae., hostname=bd6592c37bb2,40087,1732486423713, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=63842f73863e9e7b9587486b122e6aae, server=bd6592c37bb2,40087,1732486423713 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T22:14:24,109 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732486460633.63842f73863e9e7b9587486b122e6aae., hostname=bd6592c37bb2,40087,1732486423713, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=63842f73863e9e7b9587486b122e6aae, server=bd6592c37bb2,40087,1732486423713 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T22:14:24,109 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732486460633.63842f73863e9e7b9587486b122e6aae., hostname=bd6592c37bb2,40087,1732486423713, seqNum=5 because the exception is null or not the one we care about 2024-11-24T22:14:24,131 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37701 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=38 2024-11-24T22:14:24,135 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=40087 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=39 2024-11-24T22:14:24,136 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bd6592c37bb2:0-0 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732486460633.63842f73863e9e7b9587486b122e6aae. 2024-11-24T22:14:24,136 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bd6592c37bb2:0-0 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732486460633.63842f73863e9e7b9587486b122e6aae. as already flushing 2024-11-24T22:14:24,136 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bd6592c37bb2:0-0 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732486460633.63842f73863e9e7b9587486b122e6aae. 2024-11-24T22:14:24,136 ERROR [RS_FLUSH_OPERATIONS-regionserver/bd6592c37bb2:0-0 {event_type=RS_FLUSH_REGIONS, pid=39}] handler.RSProcedureHandler(58): pid=39 java.io.IOException: Unable to complete flush {ENCODED => 63842f73863e9e7b9587486b122e6aae, NAME => 'TestAcidGuarantees,,1732486460633.63842f73863e9e7b9587486b122e6aae.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T22:14:24,136 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bd6592c37bb2:0-0 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=39 java.io.IOException: Unable to complete flush {ENCODED => 63842f73863e9e7b9587486b122e6aae, NAME => 'TestAcidGuarantees,,1732486460633.63842f73863e9e7b9587486b122e6aae.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T22:14:24,137 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37701 {}] master.HMaster(4177): Remote procedure failed, pid=39 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 63842f73863e9e7b9587486b122e6aae, NAME => 'TestAcidGuarantees,,1732486460633.63842f73863e9e7b9587486b122e6aae.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 63842f73863e9e7b9587486b122e6aae, NAME => 'TestAcidGuarantees,,1732486460633.63842f73863e9e7b9587486b122e6aae.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T22:14:24,290 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=40087 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=39 2024-11-24T22:14:24,290 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bd6592c37bb2:0-1 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732486460633.63842f73863e9e7b9587486b122e6aae. 2024-11-24T22:14:24,291 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bd6592c37bb2:0-1 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732486460633.63842f73863e9e7b9587486b122e6aae. as already flushing 2024-11-24T22:14:24,291 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bd6592c37bb2:0-1 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732486460633.63842f73863e9e7b9587486b122e6aae. 2024-11-24T22:14:24,291 ERROR [RS_FLUSH_OPERATIONS-regionserver/bd6592c37bb2:0-1 {event_type=RS_FLUSH_REGIONS, pid=39}] handler.RSProcedureHandler(58): pid=39 java.io.IOException: Unable to complete flush {ENCODED => 63842f73863e9e7b9587486b122e6aae, NAME => 'TestAcidGuarantees,,1732486460633.63842f73863e9e7b9587486b122e6aae.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T22:14:24,291 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bd6592c37bb2:0-1 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=39 java.io.IOException: Unable to complete flush {ENCODED => 63842f73863e9e7b9587486b122e6aae, NAME => 'TestAcidGuarantees,,1732486460633.63842f73863e9e7b9587486b122e6aae.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T22:14:24,292 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37701 {}] master.HMaster(4177): Remote procedure failed, pid=39 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 63842f73863e9e7b9587486b122e6aae, NAME => 'TestAcidGuarantees,,1732486460633.63842f73863e9e7b9587486b122e6aae.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 63842f73863e9e7b9587486b122e6aae, NAME => 'TestAcidGuarantees,,1732486460633.63842f73863e9e7b9587486b122e6aae.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T22:14:24,323 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40087 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=63842f73863e9e7b9587486b122e6aae, server=bd6592c37bb2,40087,1732486423713 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-24T22:14:24,323 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40087 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40226 deadline: 1732486524312, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=63842f73863e9e7b9587486b122e6aae, server=bd6592c37bb2,40087,1732486423713 2024-11-24T22:14:24,324 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732486460633.63842f73863e9e7b9587486b122e6aae., hostname=bd6592c37bb2,40087,1732486423713, seqNum=5 , the old value is region=TestAcidGuarantees,,1732486460633.63842f73863e9e7b9587486b122e6aae., hostname=bd6592c37bb2,40087,1732486423713, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=63842f73863e9e7b9587486b122e6aae, server=bd6592c37bb2,40087,1732486423713 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T22:14:24,324 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732486460633.63842f73863e9e7b9587486b122e6aae., hostname=bd6592c37bb2,40087,1732486423713, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=63842f73863e9e7b9587486b122e6aae, server=bd6592c37bb2,40087,1732486423713 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T22:14:24,324 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732486460633.63842f73863e9e7b9587486b122e6aae., hostname=bd6592c37bb2,40087,1732486423713, seqNum=5 because the exception is null or not the one we care about 2024-11-24T22:14:24,324 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40087 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=63842f73863e9e7b9587486b122e6aae, server=bd6592c37bb2,40087,1732486423713 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-24T22:14:24,325 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40087 {}] ipc.CallRunner(138): callId: 12 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40206 deadline: 1732486524312, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=63842f73863e9e7b9587486b122e6aae, server=bd6592c37bb2,40087,1732486423713 2024-11-24T22:14:24,325 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732486460633.63842f73863e9e7b9587486b122e6aae., hostname=bd6592c37bb2,40087,1732486423713, seqNum=5 , the old value is region=TestAcidGuarantees,,1732486460633.63842f73863e9e7b9587486b122e6aae., hostname=bd6592c37bb2,40087,1732486423713, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=63842f73863e9e7b9587486b122e6aae, server=bd6592c37bb2,40087,1732486423713 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T22:14:24,325 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732486460633.63842f73863e9e7b9587486b122e6aae., hostname=bd6592c37bb2,40087,1732486423713, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=63842f73863e9e7b9587486b122e6aae, server=bd6592c37bb2,40087,1732486423713 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T22:14:24,325 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732486460633.63842f73863e9e7b9587486b122e6aae., hostname=bd6592c37bb2,40087,1732486423713, seqNum=5 because the exception is null or not the one we care about 2024-11-24T22:14:24,333 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40087 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=63842f73863e9e7b9587486b122e6aae, server=bd6592c37bb2,40087,1732486423713 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-24T22:14:24,334 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40087 {}] ipc.CallRunner(138): callId: 14 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40202 deadline: 1732486524313, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=63842f73863e9e7b9587486b122e6aae, server=bd6592c37bb2,40087,1732486423713 2024-11-24T22:14:24,337 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732486460633.63842f73863e9e7b9587486b122e6aae., hostname=bd6592c37bb2,40087,1732486423713, seqNum=5 , the old value is region=TestAcidGuarantees,,1732486460633.63842f73863e9e7b9587486b122e6aae., hostname=bd6592c37bb2,40087,1732486423713, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=63842f73863e9e7b9587486b122e6aae, server=bd6592c37bb2,40087,1732486423713 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T22:14:24,337 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732486460633.63842f73863e9e7b9587486b122e6aae., hostname=bd6592c37bb2,40087,1732486423713, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=63842f73863e9e7b9587486b122e6aae, server=bd6592c37bb2,40087,1732486423713 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T22:14:24,337 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732486460633.63842f73863e9e7b9587486b122e6aae., hostname=bd6592c37bb2,40087,1732486423713, seqNum=5 because the exception is null or not the one we care about 2024-11-24T22:14:24,337 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40087 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=63842f73863e9e7b9587486b122e6aae, server=bd6592c37bb2,40087,1732486423713 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-24T22:14:24,337 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40087 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=63842f73863e9e7b9587486b122e6aae, server=bd6592c37bb2,40087,1732486423713 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-24T22:14:24,338 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40087 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40244 deadline: 1732486524321, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=63842f73863e9e7b9587486b122e6aae, server=bd6592c37bb2,40087,1732486423713 2024-11-24T22:14:24,338 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732486460633.63842f73863e9e7b9587486b122e6aae., hostname=bd6592c37bb2,40087,1732486423713, seqNum=5 , the old value is region=TestAcidGuarantees,,1732486460633.63842f73863e9e7b9587486b122e6aae., hostname=bd6592c37bb2,40087,1732486423713, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=63842f73863e9e7b9587486b122e6aae, server=bd6592c37bb2,40087,1732486423713 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T22:14:24,338 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732486460633.63842f73863e9e7b9587486b122e6aae., hostname=bd6592c37bb2,40087,1732486423713, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=63842f73863e9e7b9587486b122e6aae, server=bd6592c37bb2,40087,1732486423713 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T22:14:24,338 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732486460633.63842f73863e9e7b9587486b122e6aae., hostname=bd6592c37bb2,40087,1732486423713, seqNum=5 because the exception is null or not the one we care about 2024-11-24T22:14:24,339 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40087 {}] ipc.CallRunner(138): callId: 11 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40232 deadline: 1732486524321, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=63842f73863e9e7b9587486b122e6aae, server=bd6592c37bb2,40087,1732486423713 2024-11-24T22:14:24,340 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732486460633.63842f73863e9e7b9587486b122e6aae., hostname=bd6592c37bb2,40087,1732486423713, seqNum=5 , the old value is region=TestAcidGuarantees,,1732486460633.63842f73863e9e7b9587486b122e6aae., hostname=bd6592c37bb2,40087,1732486423713, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=63842f73863e9e7b9587486b122e6aae, server=bd6592c37bb2,40087,1732486423713 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T22:14:24,340 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732486460633.63842f73863e9e7b9587486b122e6aae., hostname=bd6592c37bb2,40087,1732486423713, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=63842f73863e9e7b9587486b122e6aae, server=bd6592c37bb2,40087,1732486423713 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T22:14:24,340 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732486460633.63842f73863e9e7b9587486b122e6aae., hostname=bd6592c37bb2,40087,1732486423713, seqNum=5 because the exception is null or not the one we care about 2024-11-24T22:14:24,373 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T22:14:24,380 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411244231586579d04048854e6a51911a219c_63842f73863e9e7b9587486b122e6aae to hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411244231586579d04048854e6a51911a219c_63842f73863e9e7b9587486b122e6aae 2024-11-24T22:14:24,381 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/63842f73863e9e7b9587486b122e6aae/.tmp/A/13672022da204af48915a725fbfa98a5, store: [table=TestAcidGuarantees family=A region=63842f73863e9e7b9587486b122e6aae] 2024-11-24T22:14:24,401 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/63842f73863e9e7b9587486b122e6aae/.tmp/A/13672022da204af48915a725fbfa98a5 is 175, key is test_row_0/A:col10/1732486463857/Put/seqid=0 2024-11-24T22:14:24,440 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37701 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=38 2024-11-24T22:14:24,445 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45929 is added to blk_1073741940_1116 (size=56733) 2024-11-24T22:14:24,450 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=15, memsize=17.9 K, hasBloomFilter=true, into tmp file hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/63842f73863e9e7b9587486b122e6aae/.tmp/A/13672022da204af48915a725fbfa98a5 2024-11-24T22:14:24,453 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=40087 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=39 2024-11-24T22:14:24,453 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bd6592c37bb2:0-2 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732486460633.63842f73863e9e7b9587486b122e6aae. 2024-11-24T22:14:24,453 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bd6592c37bb2:0-2 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732486460633.63842f73863e9e7b9587486b122e6aae. as already flushing 2024-11-24T22:14:24,453 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bd6592c37bb2:0-2 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732486460633.63842f73863e9e7b9587486b122e6aae. 2024-11-24T22:14:24,454 ERROR [RS_FLUSH_OPERATIONS-regionserver/bd6592c37bb2:0-2 {event_type=RS_FLUSH_REGIONS, pid=39}] handler.RSProcedureHandler(58): pid=39 java.io.IOException: Unable to complete flush {ENCODED => 63842f73863e9e7b9587486b122e6aae, NAME => 'TestAcidGuarantees,,1732486460633.63842f73863e9e7b9587486b122e6aae.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T22:14:24,454 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bd6592c37bb2:0-2 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=39 java.io.IOException: Unable to complete flush {ENCODED => 63842f73863e9e7b9587486b122e6aae, NAME => 'TestAcidGuarantees,,1732486460633.63842f73863e9e7b9587486b122e6aae.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T22:14:24,455 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37701 {}] master.HMaster(4177): Remote procedure failed, pid=39 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 63842f73863e9e7b9587486b122e6aae, NAME => 'TestAcidGuarantees,,1732486460633.63842f73863e9e7b9587486b122e6aae.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 63842f73863e9e7b9587486b122e6aae, NAME => 'TestAcidGuarantees,,1732486460633.63842f73863e9e7b9587486b122e6aae.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T22:14:24,491 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/63842f73863e9e7b9587486b122e6aae/.tmp/B/e0d8bb8944b64909b312bea8b1cacdd8 is 50, key is test_row_0/B:col10/1732486463857/Put/seqid=0 2024-11-24T22:14:24,504 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45929 is added to blk_1073741941_1117 (size=12001) 2024-11-24T22:14:24,505 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=15 (bloomFilter=true), to=hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/63842f73863e9e7b9587486b122e6aae/.tmp/B/e0d8bb8944b64909b312bea8b1cacdd8 2024-11-24T22:14:24,542 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/63842f73863e9e7b9587486b122e6aae/.tmp/C/6711daa1d8ff424e8ac2f9cd4c7287f4 is 50, key is test_row_0/C:col10/1732486463857/Put/seqid=0 2024-11-24T22:14:24,565 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45929 is added to blk_1073741942_1118 (size=12001) 2024-11-24T22:14:24,607 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=40087 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=39 2024-11-24T22:14:24,608 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bd6592c37bb2:0-0 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732486460633.63842f73863e9e7b9587486b122e6aae. 2024-11-24T22:14:24,608 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bd6592c37bb2:0-0 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732486460633.63842f73863e9e7b9587486b122e6aae. as already flushing 2024-11-24T22:14:24,608 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bd6592c37bb2:0-0 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732486460633.63842f73863e9e7b9587486b122e6aae. 2024-11-24T22:14:24,608 ERROR [RS_FLUSH_OPERATIONS-regionserver/bd6592c37bb2:0-0 {event_type=RS_FLUSH_REGIONS, pid=39}] handler.RSProcedureHandler(58): pid=39 java.io.IOException: Unable to complete flush {ENCODED => 63842f73863e9e7b9587486b122e6aae, NAME => 'TestAcidGuarantees,,1732486460633.63842f73863e9e7b9587486b122e6aae.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T22:14:24,608 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bd6592c37bb2:0-0 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=39 java.io.IOException: Unable to complete flush {ENCODED => 63842f73863e9e7b9587486b122e6aae, NAME => 'TestAcidGuarantees,,1732486460633.63842f73863e9e7b9587486b122e6aae.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T22:14:24,609 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37701 {}] master.HMaster(4177): Remote procedure failed, pid=39 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 63842f73863e9e7b9587486b122e6aae, NAME => 'TestAcidGuarantees,,1732486460633.63842f73863e9e7b9587486b122e6aae.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 63842f73863e9e7b9587486b122e6aae, NAME => 'TestAcidGuarantees,,1732486460633.63842f73863e9e7b9587486b122e6aae.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T22:14:24,633 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40087 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=63842f73863e9e7b9587486b122e6aae, server=bd6592c37bb2,40087,1732486423713 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-24T22:14:24,633 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40087 {}] ipc.CallRunner(138): callId: 13 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40206 deadline: 1732486524631, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=63842f73863e9e7b9587486b122e6aae, server=bd6592c37bb2,40087,1732486423713 2024-11-24T22:14:24,633 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40087 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=63842f73863e9e7b9587486b122e6aae, server=bd6592c37bb2,40087,1732486423713 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-24T22:14:24,634 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732486460633.63842f73863e9e7b9587486b122e6aae., hostname=bd6592c37bb2,40087,1732486423713, seqNum=5 , the old value is region=TestAcidGuarantees,,1732486460633.63842f73863e9e7b9587486b122e6aae., hostname=bd6592c37bb2,40087,1732486423713, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=63842f73863e9e7b9587486b122e6aae, server=bd6592c37bb2,40087,1732486423713 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T22:14:24,634 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40087 {}] ipc.CallRunner(138): callId: 11 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40226 deadline: 1732486524631, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=63842f73863e9e7b9587486b122e6aae, server=bd6592c37bb2,40087,1732486423713 2024-11-24T22:14:24,634 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732486460633.63842f73863e9e7b9587486b122e6aae., hostname=bd6592c37bb2,40087,1732486423713, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=63842f73863e9e7b9587486b122e6aae, server=bd6592c37bb2,40087,1732486423713 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T22:14:24,634 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732486460633.63842f73863e9e7b9587486b122e6aae., hostname=bd6592c37bb2,40087,1732486423713, seqNum=5 because the exception is null or not the one we care about 2024-11-24T22:14:24,634 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732486460633.63842f73863e9e7b9587486b122e6aae., hostname=bd6592c37bb2,40087,1732486423713, seqNum=5 , the old value is region=TestAcidGuarantees,,1732486460633.63842f73863e9e7b9587486b122e6aae., hostname=bd6592c37bb2,40087,1732486423713, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=63842f73863e9e7b9587486b122e6aae, server=bd6592c37bb2,40087,1732486423713 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T22:14:24,634 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732486460633.63842f73863e9e7b9587486b122e6aae., hostname=bd6592c37bb2,40087,1732486423713, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=63842f73863e9e7b9587486b122e6aae, server=bd6592c37bb2,40087,1732486423713 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T22:14:24,634 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732486460633.63842f73863e9e7b9587486b122e6aae., hostname=bd6592c37bb2,40087,1732486423713, seqNum=5 because the exception is null or not the one we care about 2024-11-24T22:14:24,648 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40087 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=63842f73863e9e7b9587486b122e6aae, server=bd6592c37bb2,40087,1732486423713 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-24T22:14:24,649 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40087 {}] ipc.CallRunner(138): callId: 15 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40202 deadline: 1732486524642, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=63842f73863e9e7b9587486b122e6aae, server=bd6592c37bb2,40087,1732486423713 2024-11-24T22:14:24,649 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732486460633.63842f73863e9e7b9587486b122e6aae., hostname=bd6592c37bb2,40087,1732486423713, seqNum=5 , the old value is region=TestAcidGuarantees,,1732486460633.63842f73863e9e7b9587486b122e6aae., hostname=bd6592c37bb2,40087,1732486423713, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=63842f73863e9e7b9587486b122e6aae, server=bd6592c37bb2,40087,1732486423713 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T22:14:24,649 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732486460633.63842f73863e9e7b9587486b122e6aae., hostname=bd6592c37bb2,40087,1732486423713, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=63842f73863e9e7b9587486b122e6aae, server=bd6592c37bb2,40087,1732486423713 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T22:14:24,649 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732486460633.63842f73863e9e7b9587486b122e6aae., hostname=bd6592c37bb2,40087,1732486423713, seqNum=5 because the exception is null or not the one we care about 2024-11-24T22:14:24,655 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40087 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=63842f73863e9e7b9587486b122e6aae, server=bd6592c37bb2,40087,1732486423713 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-24T22:14:24,655 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40087 {}] ipc.CallRunner(138): callId: 12 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40232 deadline: 1732486524653, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=63842f73863e9e7b9587486b122e6aae, server=bd6592c37bb2,40087,1732486423713 2024-11-24T22:14:24,656 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732486460633.63842f73863e9e7b9587486b122e6aae., hostname=bd6592c37bb2,40087,1732486423713, seqNum=5 , the old value is region=TestAcidGuarantees,,1732486460633.63842f73863e9e7b9587486b122e6aae., hostname=bd6592c37bb2,40087,1732486423713, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=63842f73863e9e7b9587486b122e6aae, server=bd6592c37bb2,40087,1732486423713 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T22:14:24,656 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40087 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=63842f73863e9e7b9587486b122e6aae, server=bd6592c37bb2,40087,1732486423713 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-24T22:14:24,656 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732486460633.63842f73863e9e7b9587486b122e6aae., hostname=bd6592c37bb2,40087,1732486423713, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=63842f73863e9e7b9587486b122e6aae, server=bd6592c37bb2,40087,1732486423713 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T22:14:24,656 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732486460633.63842f73863e9e7b9587486b122e6aae., hostname=bd6592c37bb2,40087,1732486423713, seqNum=5 because the exception is null or not the one we care about 2024-11-24T22:14:24,656 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40087 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40244 deadline: 1732486524653, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=63842f73863e9e7b9587486b122e6aae, server=bd6592c37bb2,40087,1732486423713 2024-11-24T22:14:24,656 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732486460633.63842f73863e9e7b9587486b122e6aae., hostname=bd6592c37bb2,40087,1732486423713, seqNum=5 , the old value is region=TestAcidGuarantees,,1732486460633.63842f73863e9e7b9587486b122e6aae., hostname=bd6592c37bb2,40087,1732486423713, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=63842f73863e9e7b9587486b122e6aae, server=bd6592c37bb2,40087,1732486423713 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T22:14:24,656 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732486460633.63842f73863e9e7b9587486b122e6aae., hostname=bd6592c37bb2,40087,1732486423713, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=63842f73863e9e7b9587486b122e6aae, server=bd6592c37bb2,40087,1732486423713 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T22:14:24,656 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732486460633.63842f73863e9e7b9587486b122e6aae., hostname=bd6592c37bb2,40087,1732486423713, seqNum=5 because the exception is null or not the one we care about 2024-11-24T22:14:24,762 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=40087 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=39 2024-11-24T22:14:24,762 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bd6592c37bb2:0-1 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732486460633.63842f73863e9e7b9587486b122e6aae. 2024-11-24T22:14:24,762 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bd6592c37bb2:0-1 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732486460633.63842f73863e9e7b9587486b122e6aae. as already flushing 2024-11-24T22:14:24,762 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bd6592c37bb2:0-1 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732486460633.63842f73863e9e7b9587486b122e6aae. 2024-11-24T22:14:24,763 ERROR [RS_FLUSH_OPERATIONS-regionserver/bd6592c37bb2:0-1 {event_type=RS_FLUSH_REGIONS, pid=39}] handler.RSProcedureHandler(58): pid=39 java.io.IOException: Unable to complete flush {ENCODED => 63842f73863e9e7b9587486b122e6aae, NAME => 'TestAcidGuarantees,,1732486460633.63842f73863e9e7b9587486b122e6aae.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T22:14:24,763 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bd6592c37bb2:0-1 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=39 java.io.IOException: Unable to complete flush {ENCODED => 63842f73863e9e7b9587486b122e6aae, NAME => 'TestAcidGuarantees,,1732486460633.63842f73863e9e7b9587486b122e6aae.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T22:14:24,763 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37701 {}] master.HMaster(4177): Remote procedure failed, pid=39 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 63842f73863e9e7b9587486b122e6aae, NAME => 'TestAcidGuarantees,,1732486460633.63842f73863e9e7b9587486b122e6aae.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 63842f73863e9e7b9587486b122e6aae, NAME => 'TestAcidGuarantees,,1732486460633.63842f73863e9e7b9587486b122e6aae.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T22:14:24,920 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=40087 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=39 2024-11-24T22:14:24,920 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bd6592c37bb2:0-2 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732486460633.63842f73863e9e7b9587486b122e6aae. 2024-11-24T22:14:24,920 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bd6592c37bb2:0-2 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732486460633.63842f73863e9e7b9587486b122e6aae. as already flushing 2024-11-24T22:14:24,920 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bd6592c37bb2:0-2 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732486460633.63842f73863e9e7b9587486b122e6aae. 2024-11-24T22:14:24,920 ERROR [RS_FLUSH_OPERATIONS-regionserver/bd6592c37bb2:0-2 {event_type=RS_FLUSH_REGIONS, pid=39}] handler.RSProcedureHandler(58): pid=39 java.io.IOException: Unable to complete flush {ENCODED => 63842f73863e9e7b9587486b122e6aae, NAME => 'TestAcidGuarantees,,1732486460633.63842f73863e9e7b9587486b122e6aae.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T22:14:24,920 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bd6592c37bb2:0-2 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=39 java.io.IOException: Unable to complete flush {ENCODED => 63842f73863e9e7b9587486b122e6aae, NAME => 'TestAcidGuarantees,,1732486460633.63842f73863e9e7b9587486b122e6aae.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T22:14:24,921 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37701 {}] master.HMaster(4177): Remote procedure failed, pid=39 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 63842f73863e9e7b9587486b122e6aae, NAME => 'TestAcidGuarantees,,1732486460633.63842f73863e9e7b9587486b122e6aae.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 63842f73863e9e7b9587486b122e6aae, NAME => 'TestAcidGuarantees,,1732486460633.63842f73863e9e7b9587486b122e6aae.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T22:14:24,951 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37701 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=38 2024-11-24T22:14:24,963 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=15 (bloomFilter=true), to=hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/63842f73863e9e7b9587486b122e6aae/.tmp/C/6711daa1d8ff424e8ac2f9cd4c7287f4 2024-11-24T22:14:24,971 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/63842f73863e9e7b9587486b122e6aae/.tmp/A/13672022da204af48915a725fbfa98a5 as hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/63842f73863e9e7b9587486b122e6aae/A/13672022da204af48915a725fbfa98a5 2024-11-24T22:14:24,978 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/63842f73863e9e7b9587486b122e6aae/A/13672022da204af48915a725fbfa98a5, entries=300, sequenceid=15, filesize=55.4 K 2024-11-24T22:14:24,979 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/63842f73863e9e7b9587486b122e6aae/.tmp/B/e0d8bb8944b64909b312bea8b1cacdd8 as hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/63842f73863e9e7b9587486b122e6aae/B/e0d8bb8944b64909b312bea8b1cacdd8 2024-11-24T22:14:24,993 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/63842f73863e9e7b9587486b122e6aae/B/e0d8bb8944b64909b312bea8b1cacdd8, entries=150, sequenceid=15, filesize=11.7 K 2024-11-24T22:14:24,994 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/63842f73863e9e7b9587486b122e6aae/.tmp/C/6711daa1d8ff424e8ac2f9cd4c7287f4 as hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/63842f73863e9e7b9587486b122e6aae/C/6711daa1d8ff424e8ac2f9cd4c7287f4 2024-11-24T22:14:25,000 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/63842f73863e9e7b9587486b122e6aae/C/6711daa1d8ff424e8ac2f9cd4c7287f4, entries=150, sequenceid=15, filesize=11.7 K 2024-11-24T22:14:25,001 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~53.67 KB/54960, heapSize ~160.08 KB/163920, currentSize=127.08 KB/130125 for 63842f73863e9e7b9587486b122e6aae in 1142ms, sequenceid=15, compaction requested=false 2024-11-24T22:14:25,001 DEBUG [MemStoreFlusher.0 {}] regionserver.MetricsTableSourceImpl(133): Creating new MetricsTableSourceImpl for table 'TestAcidGuarantees' 2024-11-24T22:14:25,003 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 63842f73863e9e7b9587486b122e6aae: 2024-11-24T22:14:25,075 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=40087 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=39 2024-11-24T22:14:25,075 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bd6592c37bb2:0-0 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732486460633.63842f73863e9e7b9587486b122e6aae. 2024-11-24T22:14:25,076 INFO [RS_FLUSH_OPERATIONS-regionserver/bd6592c37bb2:0-0 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.HRegion(2902): Flushing 63842f73863e9e7b9587486b122e6aae 3/3 column families, dataSize=127.08 KB heapSize=364.15 KB 2024-11-24T22:14:25,076 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bd6592c37bb2:0-0 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 63842f73863e9e7b9587486b122e6aae, store=A 2024-11-24T22:14:25,076 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bd6592c37bb2:0-0 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-24T22:14:25,076 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bd6592c37bb2:0-0 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 63842f73863e9e7b9587486b122e6aae, store=B 2024-11-24T22:14:25,076 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bd6592c37bb2:0-0 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-24T22:14:25,076 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bd6592c37bb2:0-0 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 63842f73863e9e7b9587486b122e6aae, store=C 2024-11-24T22:14:25,076 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bd6592c37bb2:0-0 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-24T22:14:25,093 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bd6592c37bb2:0-0 {event_type=RS_FLUSH_REGIONS, pid=39}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024112400f4cd1cec9a4a14861e26f4dc88f79b_63842f73863e9e7b9587486b122e6aae is 50, key is test_row_0/A:col10/1732486463971/Put/seqid=0 2024-11-24T22:14:25,126 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45929 is added to blk_1073741943_1119 (size=12154) 2024-11-24T22:14:25,128 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bd6592c37bb2:0-0 {event_type=RS_FLUSH_REGIONS, pid=39}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T22:14:25,133 INFO [RS_FLUSH_OPERATIONS-regionserver/bd6592c37bb2:0-0 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024112400f4cd1cec9a4a14861e26f4dc88f79b_63842f73863e9e7b9587486b122e6aae to hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024112400f4cd1cec9a4a14861e26f4dc88f79b_63842f73863e9e7b9587486b122e6aae 2024-11-24T22:14:25,137 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bd6592c37bb2:0-0 {event_type=RS_FLUSH_REGIONS, pid=39}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/63842f73863e9e7b9587486b122e6aae/.tmp/A/fade54e87e5f4b1b93f1630813554439, store: [table=TestAcidGuarantees family=A region=63842f73863e9e7b9587486b122e6aae] 2024-11-24T22:14:25,138 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bd6592c37bb2:0-0 {event_type=RS_FLUSH_REGIONS, pid=39}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/63842f73863e9e7b9587486b122e6aae/.tmp/A/fade54e87e5f4b1b93f1630813554439 is 175, key is test_row_0/A:col10/1732486463971/Put/seqid=0 2024-11-24T22:14:25,150 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732486460633.63842f73863e9e7b9587486b122e6aae. as already flushing 2024-11-24T22:14:25,150 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40087 {}] regionserver.HRegion(8855): Flush requested on 63842f73863e9e7b9587486b122e6aae 2024-11-24T22:14:25,153 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45929 is added to blk_1073741944_1120 (size=30955) 2024-11-24T22:14:25,156 INFO [RS_FLUSH_OPERATIONS-regionserver/bd6592c37bb2:0-0 {event_type=RS_FLUSH_REGIONS, pid=39}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=51, memsize=42.1 K, hasBloomFilter=true, into tmp file hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/63842f73863e9e7b9587486b122e6aae/.tmp/A/fade54e87e5f4b1b93f1630813554439 2024-11-24T22:14:25,168 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bd6592c37bb2:0-0 {event_type=RS_FLUSH_REGIONS, pid=39}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/63842f73863e9e7b9587486b122e6aae/.tmp/B/02bd6ba0eecd48b99e5ad15a050a2011 is 50, key is test_row_0/B:col10/1732486463971/Put/seqid=0 2024-11-24T22:14:25,177 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40087 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=63842f73863e9e7b9587486b122e6aae, server=bd6592c37bb2,40087,1732486423713 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-24T22:14:25,178 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40087 {}] ipc.CallRunner(138): callId: 17 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40206 deadline: 1732486525171, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=63842f73863e9e7b9587486b122e6aae, server=bd6592c37bb2,40087,1732486423713 2024-11-24T22:14:25,178 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732486460633.63842f73863e9e7b9587486b122e6aae., hostname=bd6592c37bb2,40087,1732486423713, seqNum=5 , the old value is region=TestAcidGuarantees,,1732486460633.63842f73863e9e7b9587486b122e6aae., hostname=bd6592c37bb2,40087,1732486423713, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=63842f73863e9e7b9587486b122e6aae, server=bd6592c37bb2,40087,1732486423713 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T22:14:25,178 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732486460633.63842f73863e9e7b9587486b122e6aae., hostname=bd6592c37bb2,40087,1732486423713, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=63842f73863e9e7b9587486b122e6aae, server=bd6592c37bb2,40087,1732486423713 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T22:14:25,178 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732486460633.63842f73863e9e7b9587486b122e6aae., hostname=bd6592c37bb2,40087,1732486423713, seqNum=5 because the exception is null or not the one we care about 2024-11-24T22:14:25,179 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40087 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=63842f73863e9e7b9587486b122e6aae, server=bd6592c37bb2,40087,1732486423713 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-24T22:14:25,179 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40087 {}] ipc.CallRunner(138): callId: 17 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40202 deadline: 1732486525173, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=63842f73863e9e7b9587486b122e6aae, server=bd6592c37bb2,40087,1732486423713 2024-11-24T22:14:25,179 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40087 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=63842f73863e9e7b9587486b122e6aae, server=bd6592c37bb2,40087,1732486423713 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-24T22:14:25,179 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732486460633.63842f73863e9e7b9587486b122e6aae., hostname=bd6592c37bb2,40087,1732486423713, seqNum=5 , the old value is region=TestAcidGuarantees,,1732486460633.63842f73863e9e7b9587486b122e6aae., hostname=bd6592c37bb2,40087,1732486423713, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=63842f73863e9e7b9587486b122e6aae, server=bd6592c37bb2,40087,1732486423713 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T22:14:25,179 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732486460633.63842f73863e9e7b9587486b122e6aae., hostname=bd6592c37bb2,40087,1732486423713, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=63842f73863e9e7b9587486b122e6aae, server=bd6592c37bb2,40087,1732486423713 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T22:14:25,179 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732486460633.63842f73863e9e7b9587486b122e6aae., hostname=bd6592c37bb2,40087,1732486423713, seqNum=5 because the exception is null or not the one we care about 2024-11-24T22:14:25,179 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40087 {}] ipc.CallRunner(138): callId: 14 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40232 deadline: 1732486525175, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=63842f73863e9e7b9587486b122e6aae, server=bd6592c37bb2,40087,1732486423713 2024-11-24T22:14:25,180 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732486460633.63842f73863e9e7b9587486b122e6aae., hostname=bd6592c37bb2,40087,1732486423713, seqNum=5 , the old value is region=TestAcidGuarantees,,1732486460633.63842f73863e9e7b9587486b122e6aae., hostname=bd6592c37bb2,40087,1732486423713, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=63842f73863e9e7b9587486b122e6aae, server=bd6592c37bb2,40087,1732486423713 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T22:14:25,180 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732486460633.63842f73863e9e7b9587486b122e6aae., hostname=bd6592c37bb2,40087,1732486423713, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=63842f73863e9e7b9587486b122e6aae, server=bd6592c37bb2,40087,1732486423713 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T22:14:25,180 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732486460633.63842f73863e9e7b9587486b122e6aae., hostname=bd6592c37bb2,40087,1732486423713, seqNum=5 because the exception is null or not the one we care about 2024-11-24T22:14:25,182 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40087 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=63842f73863e9e7b9587486b122e6aae, server=bd6592c37bb2,40087,1732486423713 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-24T22:14:25,182 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40087 {}] ipc.CallRunner(138): callId: 12 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40244 deadline: 1732486525177, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=63842f73863e9e7b9587486b122e6aae, server=bd6592c37bb2,40087,1732486423713 2024-11-24T22:14:25,183 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732486460633.63842f73863e9e7b9587486b122e6aae., hostname=bd6592c37bb2,40087,1732486423713, seqNum=5 , the old value is region=TestAcidGuarantees,,1732486460633.63842f73863e9e7b9587486b122e6aae., hostname=bd6592c37bb2,40087,1732486423713, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=63842f73863e9e7b9587486b122e6aae, server=bd6592c37bb2,40087,1732486423713 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T22:14:25,183 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732486460633.63842f73863e9e7b9587486b122e6aae., hostname=bd6592c37bb2,40087,1732486423713, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=63842f73863e9e7b9587486b122e6aae, server=bd6592c37bb2,40087,1732486423713 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T22:14:25,183 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732486460633.63842f73863e9e7b9587486b122e6aae., hostname=bd6592c37bb2,40087,1732486423713, seqNum=5 because the exception is null or not the one we care about 2024-11-24T22:14:25,183 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40087 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=63842f73863e9e7b9587486b122e6aae, server=bd6592c37bb2,40087,1732486423713 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-24T22:14:25,183 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40087 {}] ipc.CallRunner(138): callId: 15 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40226 deadline: 1732486525178, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=63842f73863e9e7b9587486b122e6aae, server=bd6592c37bb2,40087,1732486423713 2024-11-24T22:14:25,184 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732486460633.63842f73863e9e7b9587486b122e6aae., hostname=bd6592c37bb2,40087,1732486423713, seqNum=5 , the old value is region=TestAcidGuarantees,,1732486460633.63842f73863e9e7b9587486b122e6aae., hostname=bd6592c37bb2,40087,1732486423713, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=63842f73863e9e7b9587486b122e6aae, server=bd6592c37bb2,40087,1732486423713 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T22:14:25,184 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732486460633.63842f73863e9e7b9587486b122e6aae., hostname=bd6592c37bb2,40087,1732486423713, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=63842f73863e9e7b9587486b122e6aae, server=bd6592c37bb2,40087,1732486423713 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T22:14:25,184 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732486460633.63842f73863e9e7b9587486b122e6aae., hostname=bd6592c37bb2,40087,1732486423713, seqNum=5 because the exception is null or not the one we care about 2024-11-24T22:14:25,192 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45929 is added to blk_1073741945_1121 (size=12001) 2024-11-24T22:14:25,194 INFO [RS_FLUSH_OPERATIONS-regionserver/bd6592c37bb2:0-0 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=42.49 KB at sequenceid=51 (bloomFilter=true), to=hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/63842f73863e9e7b9587486b122e6aae/.tmp/B/02bd6ba0eecd48b99e5ad15a050a2011 2024-11-24T22:14:25,215 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bd6592c37bb2:0-0 {event_type=RS_FLUSH_REGIONS, pid=39}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/63842f73863e9e7b9587486b122e6aae/.tmp/C/f967706d7c7a4b2aa41d0ce3d33ee73e is 50, key is test_row_0/C:col10/1732486463971/Put/seqid=0 2024-11-24T22:14:25,230 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45929 is added to blk_1073741946_1122 (size=12001) 2024-11-24T22:14:25,282 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40087 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=63842f73863e9e7b9587486b122e6aae, server=bd6592c37bb2,40087,1732486423713 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-24T22:14:25,282 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40087 {}] ipc.CallRunner(138): callId: 18 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40206 deadline: 1732486525281, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=63842f73863e9e7b9587486b122e6aae, server=bd6592c37bb2,40087,1732486423713 2024-11-24T22:14:25,282 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40087 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=63842f73863e9e7b9587486b122e6aae, server=bd6592c37bb2,40087,1732486423713 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-24T22:14:25,282 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732486460633.63842f73863e9e7b9587486b122e6aae., hostname=bd6592c37bb2,40087,1732486423713, seqNum=5 , the old value is region=TestAcidGuarantees,,1732486460633.63842f73863e9e7b9587486b122e6aae., hostname=bd6592c37bb2,40087,1732486423713, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=63842f73863e9e7b9587486b122e6aae, server=bd6592c37bb2,40087,1732486423713 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T22:14:25,283 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732486460633.63842f73863e9e7b9587486b122e6aae., hostname=bd6592c37bb2,40087,1732486423713, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=63842f73863e9e7b9587486b122e6aae, server=bd6592c37bb2,40087,1732486423713 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T22:14:25,283 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732486460633.63842f73863e9e7b9587486b122e6aae., hostname=bd6592c37bb2,40087,1732486423713, seqNum=5 because the exception is null or not the one we care about 2024-11-24T22:14:25,283 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40087 {}] ipc.CallRunner(138): callId: 18 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40202 deadline: 1732486525281, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=63842f73863e9e7b9587486b122e6aae, server=bd6592c37bb2,40087,1732486423713 2024-11-24T22:14:25,283 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732486460633.63842f73863e9e7b9587486b122e6aae., hostname=bd6592c37bb2,40087,1732486423713, seqNum=5 , the old value is region=TestAcidGuarantees,,1732486460633.63842f73863e9e7b9587486b122e6aae., hostname=bd6592c37bb2,40087,1732486423713, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=63842f73863e9e7b9587486b122e6aae, server=bd6592c37bb2,40087,1732486423713 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T22:14:25,283 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732486460633.63842f73863e9e7b9587486b122e6aae., hostname=bd6592c37bb2,40087,1732486423713, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=63842f73863e9e7b9587486b122e6aae, server=bd6592c37bb2,40087,1732486423713 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T22:14:25,283 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732486460633.63842f73863e9e7b9587486b122e6aae., hostname=bd6592c37bb2,40087,1732486423713, seqNum=5 because the exception is null or not the one we care about 2024-11-24T22:14:25,291 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40087 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=63842f73863e9e7b9587486b122e6aae, server=bd6592c37bb2,40087,1732486423713 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-24T22:14:25,292 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40087 {}] ipc.CallRunner(138): callId: 13 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40244 deadline: 1732486525291, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=63842f73863e9e7b9587486b122e6aae, server=bd6592c37bb2,40087,1732486423713 2024-11-24T22:14:25,293 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40087 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=63842f73863e9e7b9587486b122e6aae, server=bd6592c37bb2,40087,1732486423713 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-24T22:14:25,293 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40087 {}] ipc.CallRunner(138): callId: 16 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40226 deadline: 1732486525291, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=63842f73863e9e7b9587486b122e6aae, server=bd6592c37bb2,40087,1732486423713 2024-11-24T22:14:25,293 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40087 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=63842f73863e9e7b9587486b122e6aae, server=bd6592c37bb2,40087,1732486423713 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-24T22:14:25,293 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732486460633.63842f73863e9e7b9587486b122e6aae., hostname=bd6592c37bb2,40087,1732486423713, seqNum=5 , the old value is region=TestAcidGuarantees,,1732486460633.63842f73863e9e7b9587486b122e6aae., hostname=bd6592c37bb2,40087,1732486423713, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=63842f73863e9e7b9587486b122e6aae, server=bd6592c37bb2,40087,1732486423713 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T22:14:25,293 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732486460633.63842f73863e9e7b9587486b122e6aae., hostname=bd6592c37bb2,40087,1732486423713, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=63842f73863e9e7b9587486b122e6aae, server=bd6592c37bb2,40087,1732486423713 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T22:14:25,293 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40087 {}] ipc.CallRunner(138): callId: 15 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40232 deadline: 1732486525291, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=63842f73863e9e7b9587486b122e6aae, server=bd6592c37bb2,40087,1732486423713 2024-11-24T22:14:25,293 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732486460633.63842f73863e9e7b9587486b122e6aae., hostname=bd6592c37bb2,40087,1732486423713, seqNum=5 because the exception is null or not the one we care about 2024-11-24T22:14:25,294 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732486460633.63842f73863e9e7b9587486b122e6aae., hostname=bd6592c37bb2,40087,1732486423713, seqNum=5 , the old value is region=TestAcidGuarantees,,1732486460633.63842f73863e9e7b9587486b122e6aae., hostname=bd6592c37bb2,40087,1732486423713, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=63842f73863e9e7b9587486b122e6aae, server=bd6592c37bb2,40087,1732486423713 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T22:14:25,294 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732486460633.63842f73863e9e7b9587486b122e6aae., hostname=bd6592c37bb2,40087,1732486423713, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=63842f73863e9e7b9587486b122e6aae, server=bd6592c37bb2,40087,1732486423713 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T22:14:25,294 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732486460633.63842f73863e9e7b9587486b122e6aae., hostname=bd6592c37bb2,40087,1732486423713, seqNum=5 because the exception is null or not the one we care about 2024-11-24T22:14:25,296 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732486460633.63842f73863e9e7b9587486b122e6aae., hostname=bd6592c37bb2,40087,1732486423713, seqNum=5 , the old value is region=TestAcidGuarantees,,1732486460633.63842f73863e9e7b9587486b122e6aae., hostname=bd6592c37bb2,40087,1732486423713, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=63842f73863e9e7b9587486b122e6aae, server=bd6592c37bb2,40087,1732486423713 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T22:14:25,296 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732486460633.63842f73863e9e7b9587486b122e6aae., hostname=bd6592c37bb2,40087,1732486423713, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=63842f73863e9e7b9587486b122e6aae, server=bd6592c37bb2,40087,1732486423713 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T22:14:25,296 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732486460633.63842f73863e9e7b9587486b122e6aae., hostname=bd6592c37bb2,40087,1732486423713, seqNum=5 because the exception is null or not the one we care about 2024-11-24T22:14:25,454 WARN [HBase-Metrics2-1 {}] impl.MetricsConfig(138): Cannot locate configuration: tried hadoop-metrics2-hbase.properties,hadoop-metrics2.properties 2024-11-24T22:14:25,493 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40087 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=63842f73863e9e7b9587486b122e6aae, server=bd6592c37bb2,40087,1732486423713 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-24T22:14:25,493 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40087 {}] ipc.CallRunner(138): callId: 19 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40202 deadline: 1732486525491, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=63842f73863e9e7b9587486b122e6aae, server=bd6592c37bb2,40087,1732486423713 2024-11-24T22:14:25,493 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40087 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=63842f73863e9e7b9587486b122e6aae, server=bd6592c37bb2,40087,1732486423713 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-24T22:14:25,494 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732486460633.63842f73863e9e7b9587486b122e6aae., hostname=bd6592c37bb2,40087,1732486423713, seqNum=5 , the old value is region=TestAcidGuarantees,,1732486460633.63842f73863e9e7b9587486b122e6aae., hostname=bd6592c37bb2,40087,1732486423713, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=63842f73863e9e7b9587486b122e6aae, server=bd6592c37bb2,40087,1732486423713 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T22:14:25,494 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732486460633.63842f73863e9e7b9587486b122e6aae., hostname=bd6592c37bb2,40087,1732486423713, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=63842f73863e9e7b9587486b122e6aae, server=bd6592c37bb2,40087,1732486423713 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T22:14:25,494 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732486460633.63842f73863e9e7b9587486b122e6aae., hostname=bd6592c37bb2,40087,1732486423713, seqNum=5 because the exception is null or not the one we care about 2024-11-24T22:14:25,494 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40087 {}] ipc.CallRunner(138): callId: 19 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40206 deadline: 1732486525491, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=63842f73863e9e7b9587486b122e6aae, server=bd6592c37bb2,40087,1732486423713 2024-11-24T22:14:25,494 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732486460633.63842f73863e9e7b9587486b122e6aae., hostname=bd6592c37bb2,40087,1732486423713, seqNum=5 , the old value is region=TestAcidGuarantees,,1732486460633.63842f73863e9e7b9587486b122e6aae., hostname=bd6592c37bb2,40087,1732486423713, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=63842f73863e9e7b9587486b122e6aae, server=bd6592c37bb2,40087,1732486423713 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T22:14:25,494 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732486460633.63842f73863e9e7b9587486b122e6aae., hostname=bd6592c37bb2,40087,1732486423713, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=63842f73863e9e7b9587486b122e6aae, server=bd6592c37bb2,40087,1732486423713 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T22:14:25,494 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732486460633.63842f73863e9e7b9587486b122e6aae., hostname=bd6592c37bb2,40087,1732486423713, seqNum=5 because the exception is null or not the one we care about 2024-11-24T22:14:25,511 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40087 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=63842f73863e9e7b9587486b122e6aae, server=bd6592c37bb2,40087,1732486423713 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-24T22:14:25,511 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40087 {}] ipc.CallRunner(138): callId: 17 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40226 deadline: 1732486525501, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=63842f73863e9e7b9587486b122e6aae, server=bd6592c37bb2,40087,1732486423713 2024-11-24T22:14:25,512 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40087 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=63842f73863e9e7b9587486b122e6aae, server=bd6592c37bb2,40087,1732486423713 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-24T22:14:25,512 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732486460633.63842f73863e9e7b9587486b122e6aae., hostname=bd6592c37bb2,40087,1732486423713, seqNum=5 , the old value is region=TestAcidGuarantees,,1732486460633.63842f73863e9e7b9587486b122e6aae., hostname=bd6592c37bb2,40087,1732486423713, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=63842f73863e9e7b9587486b122e6aae, server=bd6592c37bb2,40087,1732486423713 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T22:14:25,512 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732486460633.63842f73863e9e7b9587486b122e6aae., hostname=bd6592c37bb2,40087,1732486423713, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=63842f73863e9e7b9587486b122e6aae, server=bd6592c37bb2,40087,1732486423713 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T22:14:25,512 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40087 {}] ipc.CallRunner(138): callId: 16 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40232 deadline: 1732486525501, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=63842f73863e9e7b9587486b122e6aae, server=bd6592c37bb2,40087,1732486423713 2024-11-24T22:14:25,512 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732486460633.63842f73863e9e7b9587486b122e6aae., hostname=bd6592c37bb2,40087,1732486423713, seqNum=5 because the exception is null or not the one we care about 2024-11-24T22:14:25,512 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40087 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=63842f73863e9e7b9587486b122e6aae, server=bd6592c37bb2,40087,1732486423713 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-24T22:14:25,512 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40087 {}] ipc.CallRunner(138): callId: 14 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40244 deadline: 1732486525501, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=63842f73863e9e7b9587486b122e6aae, server=bd6592c37bb2,40087,1732486423713 2024-11-24T22:14:25,513 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732486460633.63842f73863e9e7b9587486b122e6aae., hostname=bd6592c37bb2,40087,1732486423713, seqNum=5 , the old value is region=TestAcidGuarantees,,1732486460633.63842f73863e9e7b9587486b122e6aae., hostname=bd6592c37bb2,40087,1732486423713, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=63842f73863e9e7b9587486b122e6aae, server=bd6592c37bb2,40087,1732486423713 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T22:14:25,513 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732486460633.63842f73863e9e7b9587486b122e6aae., hostname=bd6592c37bb2,40087,1732486423713, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=63842f73863e9e7b9587486b122e6aae, server=bd6592c37bb2,40087,1732486423713 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T22:14:25,513 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732486460633.63842f73863e9e7b9587486b122e6aae., hostname=bd6592c37bb2,40087,1732486423713, seqNum=5 because the exception is null or not the one we care about 2024-11-24T22:14:25,513 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732486460633.63842f73863e9e7b9587486b122e6aae., hostname=bd6592c37bb2,40087,1732486423713, seqNum=5 , the old value is region=TestAcidGuarantees,,1732486460633.63842f73863e9e7b9587486b122e6aae., hostname=bd6592c37bb2,40087,1732486423713, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=63842f73863e9e7b9587486b122e6aae, server=bd6592c37bb2,40087,1732486423713 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T22:14:25,513 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732486460633.63842f73863e9e7b9587486b122e6aae., hostname=bd6592c37bb2,40087,1732486423713, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=63842f73863e9e7b9587486b122e6aae, server=bd6592c37bb2,40087,1732486423713 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T22:14:25,513 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732486460633.63842f73863e9e7b9587486b122e6aae., hostname=bd6592c37bb2,40087,1732486423713, seqNum=5 because the exception is null or not the one we care about 2024-11-24T22:14:25,632 INFO [RS_FLUSH_OPERATIONS-regionserver/bd6592c37bb2:0-0 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=42.49 KB at sequenceid=51 (bloomFilter=true), to=hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/63842f73863e9e7b9587486b122e6aae/.tmp/C/f967706d7c7a4b2aa41d0ce3d33ee73e 2024-11-24T22:14:25,639 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bd6592c37bb2:0-0 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/63842f73863e9e7b9587486b122e6aae/.tmp/A/fade54e87e5f4b1b93f1630813554439 as hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/63842f73863e9e7b9587486b122e6aae/A/fade54e87e5f4b1b93f1630813554439 2024-11-24T22:14:25,649 INFO [RS_FLUSH_OPERATIONS-regionserver/bd6592c37bb2:0-0 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/63842f73863e9e7b9587486b122e6aae/A/fade54e87e5f4b1b93f1630813554439, entries=150, sequenceid=51, filesize=30.2 K 2024-11-24T22:14:25,650 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bd6592c37bb2:0-0 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/63842f73863e9e7b9587486b122e6aae/.tmp/B/02bd6ba0eecd48b99e5ad15a050a2011 as hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/63842f73863e9e7b9587486b122e6aae/B/02bd6ba0eecd48b99e5ad15a050a2011 2024-11-24T22:14:25,657 INFO [RS_FLUSH_OPERATIONS-regionserver/bd6592c37bb2:0-0 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/63842f73863e9e7b9587486b122e6aae/B/02bd6ba0eecd48b99e5ad15a050a2011, entries=150, sequenceid=51, filesize=11.7 K 2024-11-24T22:14:25,659 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bd6592c37bb2:0-0 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/63842f73863e9e7b9587486b122e6aae/.tmp/C/f967706d7c7a4b2aa41d0ce3d33ee73e as hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/63842f73863e9e7b9587486b122e6aae/C/f967706d7c7a4b2aa41d0ce3d33ee73e 2024-11-24T22:14:25,665 INFO [RS_FLUSH_OPERATIONS-regionserver/bd6592c37bb2:0-0 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/63842f73863e9e7b9587486b122e6aae/C/f967706d7c7a4b2aa41d0ce3d33ee73e, entries=150, sequenceid=51, filesize=11.7 K 2024-11-24T22:14:25,666 INFO [RS_FLUSH_OPERATIONS-regionserver/bd6592c37bb2:0-0 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.HRegion(3140): Finished flush of dataSize ~127.08 KB/130125, heapSize ~364.10 KB/372840, currentSize=60.38 KB/61830 for 63842f73863e9e7b9587486b122e6aae in 590ms, sequenceid=51, compaction requested=false 2024-11-24T22:14:25,667 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bd6592c37bb2:0-0 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.HRegion(2603): Flush status journal for 63842f73863e9e7b9587486b122e6aae: 2024-11-24T22:14:25,667 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bd6592c37bb2:0-0 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732486460633.63842f73863e9e7b9587486b122e6aae. 2024-11-24T22:14:25,667 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bd6592c37bb2:0-0 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=39 2024-11-24T22:14:25,667 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37701 {}] master.HMaster(4169): Remote procedure done, pid=39 2024-11-24T22:14:25,674 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=39, resume processing ppid=38 2024-11-24T22:14:25,674 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=39, ppid=38, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.8480 sec 2024-11-24T22:14:25,676 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=38, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=38, table=TestAcidGuarantees in 1.8580 sec 2024-11-24T22:14:25,809 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 63842f73863e9e7b9587486b122e6aae 3/3 column families, dataSize=73.80 KB heapSize=219.89 KB 2024-11-24T22:14:25,809 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 63842f73863e9e7b9587486b122e6aae, store=A 2024-11-24T22:14:25,809 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-24T22:14:25,809 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 63842f73863e9e7b9587486b122e6aae, store=B 2024-11-24T22:14:25,809 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-24T22:14:25,809 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 63842f73863e9e7b9587486b122e6aae, store=C 2024-11-24T22:14:25,809 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-24T22:14:25,810 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40087 {}] regionserver.HRegion(8855): Flush requested on 63842f73863e9e7b9587486b122e6aae 2024-11-24T22:14:25,845 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241124fae42328bfa745698e2372ca001493d4_63842f73863e9e7b9587486b122e6aae is 50, key is test_row_0/A:col10/1732486465806/Put/seqid=0 2024-11-24T22:14:25,873 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45929 is added to blk_1073741947_1123 (size=17034) 2024-11-24T22:14:25,874 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T22:14:25,876 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40087 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=63842f73863e9e7b9587486b122e6aae, server=bd6592c37bb2,40087,1732486423713 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-24T22:14:25,877 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40087 {}] ipc.CallRunner(138): callId: 24 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40202 deadline: 1732486525861, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=63842f73863e9e7b9587486b122e6aae, server=bd6592c37bb2,40087,1732486423713 2024-11-24T22:14:25,877 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732486460633.63842f73863e9e7b9587486b122e6aae., hostname=bd6592c37bb2,40087,1732486423713, seqNum=5 , the old value is region=TestAcidGuarantees,,1732486460633.63842f73863e9e7b9587486b122e6aae., hostname=bd6592c37bb2,40087,1732486423713, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=63842f73863e9e7b9587486b122e6aae, server=bd6592c37bb2,40087,1732486423713 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T22:14:25,877 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732486460633.63842f73863e9e7b9587486b122e6aae., hostname=bd6592c37bb2,40087,1732486423713, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=63842f73863e9e7b9587486b122e6aae, server=bd6592c37bb2,40087,1732486423713 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T22:14:25,877 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732486460633.63842f73863e9e7b9587486b122e6aae., hostname=bd6592c37bb2,40087,1732486423713, seqNum=5 because the exception is null or not the one we care about 2024-11-24T22:14:25,885 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241124fae42328bfa745698e2372ca001493d4_63842f73863e9e7b9587486b122e6aae to hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241124fae42328bfa745698e2372ca001493d4_63842f73863e9e7b9587486b122e6aae 2024-11-24T22:14:25,887 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/63842f73863e9e7b9587486b122e6aae/.tmp/A/58aff1a92da2493ab1bf579d455a3632, store: [table=TestAcidGuarantees family=A region=63842f73863e9e7b9587486b122e6aae] 2024-11-24T22:14:25,888 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/63842f73863e9e7b9587486b122e6aae/.tmp/A/58aff1a92da2493ab1bf579d455a3632 is 175, key is test_row_0/A:col10/1732486465806/Put/seqid=0 2024-11-24T22:14:25,891 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40087 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=63842f73863e9e7b9587486b122e6aae, server=bd6592c37bb2,40087,1732486423713 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-24T22:14:25,893 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40087 {}] ipc.CallRunner(138): callId: 24 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40206 deadline: 1732486525872, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=63842f73863e9e7b9587486b122e6aae, server=bd6592c37bb2,40087,1732486423713 2024-11-24T22:14:25,893 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40087 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=63842f73863e9e7b9587486b122e6aae, server=bd6592c37bb2,40087,1732486423713 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-24T22:14:25,893 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40087 {}] ipc.CallRunner(138): callId: 21 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40226 deadline: 1732486525875, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=63842f73863e9e7b9587486b122e6aae, server=bd6592c37bb2,40087,1732486423713 2024-11-24T22:14:25,893 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40087 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=63842f73863e9e7b9587486b122e6aae, server=bd6592c37bb2,40087,1732486423713 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-24T22:14:25,893 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40087 {}] ipc.CallRunner(138): callId: 18 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40244 deadline: 1732486525875, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=63842f73863e9e7b9587486b122e6aae, server=bd6592c37bb2,40087,1732486423713 2024-11-24T22:14:25,893 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732486460633.63842f73863e9e7b9587486b122e6aae., hostname=bd6592c37bb2,40087,1732486423713, seqNum=5 , the old value is region=TestAcidGuarantees,,1732486460633.63842f73863e9e7b9587486b122e6aae., hostname=bd6592c37bb2,40087,1732486423713, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=63842f73863e9e7b9587486b122e6aae, server=bd6592c37bb2,40087,1732486423713 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T22:14:25,894 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732486460633.63842f73863e9e7b9587486b122e6aae., hostname=bd6592c37bb2,40087,1732486423713, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=63842f73863e9e7b9587486b122e6aae, server=bd6592c37bb2,40087,1732486423713 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T22:14:25,893 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40087 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=63842f73863e9e7b9587486b122e6aae, server=bd6592c37bb2,40087,1732486423713 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-24T22:14:25,894 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732486460633.63842f73863e9e7b9587486b122e6aae., hostname=bd6592c37bb2,40087,1732486423713, seqNum=5 because the exception is null or not the one we care about 2024-11-24T22:14:25,894 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40087 {}] ipc.CallRunner(138): callId: 20 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40232 deadline: 1732486525877, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=63842f73863e9e7b9587486b122e6aae, server=bd6592c37bb2,40087,1732486423713 2024-11-24T22:14:25,894 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732486460633.63842f73863e9e7b9587486b122e6aae., hostname=bd6592c37bb2,40087,1732486423713, seqNum=5 , the old value is region=TestAcidGuarantees,,1732486460633.63842f73863e9e7b9587486b122e6aae., hostname=bd6592c37bb2,40087,1732486423713, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=63842f73863e9e7b9587486b122e6aae, server=bd6592c37bb2,40087,1732486423713 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T22:14:25,894 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732486460633.63842f73863e9e7b9587486b122e6aae., hostname=bd6592c37bb2,40087,1732486423713, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=63842f73863e9e7b9587486b122e6aae, server=bd6592c37bb2,40087,1732486423713 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T22:14:25,894 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732486460633.63842f73863e9e7b9587486b122e6aae., hostname=bd6592c37bb2,40087,1732486423713, seqNum=5 because the exception is null or not the one we care about 2024-11-24T22:14:25,894 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732486460633.63842f73863e9e7b9587486b122e6aae., hostname=bd6592c37bb2,40087,1732486423713, seqNum=5 , the old value is region=TestAcidGuarantees,,1732486460633.63842f73863e9e7b9587486b122e6aae., hostname=bd6592c37bb2,40087,1732486423713, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=63842f73863e9e7b9587486b122e6aae, server=bd6592c37bb2,40087,1732486423713 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T22:14:25,894 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732486460633.63842f73863e9e7b9587486b122e6aae., hostname=bd6592c37bb2,40087,1732486423713, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=63842f73863e9e7b9587486b122e6aae, server=bd6592c37bb2,40087,1732486423713 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T22:14:25,894 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732486460633.63842f73863e9e7b9587486b122e6aae., hostname=bd6592c37bb2,40087,1732486423713, seqNum=5 because the exception is null or not the one we care about 2024-11-24T22:14:25,893 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732486460633.63842f73863e9e7b9587486b122e6aae., hostname=bd6592c37bb2,40087,1732486423713, seqNum=5 , the old value is region=TestAcidGuarantees,,1732486460633.63842f73863e9e7b9587486b122e6aae., hostname=bd6592c37bb2,40087,1732486423713, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=63842f73863e9e7b9587486b122e6aae, server=bd6592c37bb2,40087,1732486423713 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T22:14:25,897 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732486460633.63842f73863e9e7b9587486b122e6aae., hostname=bd6592c37bb2,40087,1732486423713, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=63842f73863e9e7b9587486b122e6aae, server=bd6592c37bb2,40087,1732486423713 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T22:14:25,897 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732486460633.63842f73863e9e7b9587486b122e6aae., hostname=bd6592c37bb2,40087,1732486423713, seqNum=5 because the exception is null or not the one we care about 2024-11-24T22:14:25,932 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45929 is added to blk_1073741948_1124 (size=48139) 2024-11-24T22:14:25,961 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37701 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=38 2024-11-24T22:14:25,961 INFO [RPCClient-NioEventLoopGroup-4-7 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-24T22:14:25,962 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37701 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-24T22:14:25,963 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37701 {}] procedure2.ProcedureExecutor(1139): Stored pid=40, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=40, table=TestAcidGuarantees 2024-11-24T22:14:25,964 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37701 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=40 2024-11-24T22:14:25,965 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=40, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=40, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-24T22:14:25,967 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=40, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=40, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-24T22:14:25,967 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=41, ppid=40, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-24T22:14:25,981 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40087 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=63842f73863e9e7b9587486b122e6aae, server=bd6592c37bb2,40087,1732486423713 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-24T22:14:25,984 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40087 {}] ipc.CallRunner(138): callId: 25 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40202 deadline: 1732486525981, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=63842f73863e9e7b9587486b122e6aae, server=bd6592c37bb2,40087,1732486423713 2024-11-24T22:14:25,985 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732486460633.63842f73863e9e7b9587486b122e6aae., hostname=bd6592c37bb2,40087,1732486423713, seqNum=5 , the old value is region=TestAcidGuarantees,,1732486460633.63842f73863e9e7b9587486b122e6aae., hostname=bd6592c37bb2,40087,1732486423713, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=63842f73863e9e7b9587486b122e6aae, server=bd6592c37bb2,40087,1732486423713 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T22:14:25,985 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732486460633.63842f73863e9e7b9587486b122e6aae., hostname=bd6592c37bb2,40087,1732486423713, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=63842f73863e9e7b9587486b122e6aae, server=bd6592c37bb2,40087,1732486423713 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T22:14:25,985 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732486460633.63842f73863e9e7b9587486b122e6aae., hostname=bd6592c37bb2,40087,1732486423713, seqNum=5 because the exception is null or not the one we care about 2024-11-24T22:14:26,004 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40087 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=63842f73863e9e7b9587486b122e6aae, server=bd6592c37bb2,40087,1732486423713 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-24T22:14:26,004 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40087 {}] ipc.CallRunner(138): callId: 22 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40226 deadline: 1732486526001, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=63842f73863e9e7b9587486b122e6aae, server=bd6592c37bb2,40087,1732486423713 2024-11-24T22:14:26,005 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732486460633.63842f73863e9e7b9587486b122e6aae., hostname=bd6592c37bb2,40087,1732486423713, seqNum=5 , the old value is region=TestAcidGuarantees,,1732486460633.63842f73863e9e7b9587486b122e6aae., hostname=bd6592c37bb2,40087,1732486423713, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=63842f73863e9e7b9587486b122e6aae, server=bd6592c37bb2,40087,1732486423713 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T22:14:26,005 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732486460633.63842f73863e9e7b9587486b122e6aae., hostname=bd6592c37bb2,40087,1732486423713, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=63842f73863e9e7b9587486b122e6aae, server=bd6592c37bb2,40087,1732486423713 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T22:14:26,005 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732486460633.63842f73863e9e7b9587486b122e6aae., hostname=bd6592c37bb2,40087,1732486423713, seqNum=5 because the exception is null or not the one we care about 2024-11-24T22:14:26,005 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40087 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=63842f73863e9e7b9587486b122e6aae, server=bd6592c37bb2,40087,1732486423713 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-24T22:14:26,006 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40087 {}] ipc.CallRunner(138): callId: 25 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40206 deadline: 1732486526001, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=63842f73863e9e7b9587486b122e6aae, server=bd6592c37bb2,40087,1732486423713 2024-11-24T22:14:26,006 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40087 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=63842f73863e9e7b9587486b122e6aae, server=bd6592c37bb2,40087,1732486423713 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-24T22:14:26,006 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40087 {}] ipc.CallRunner(138): callId: 21 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40232 deadline: 1732486526001, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=63842f73863e9e7b9587486b122e6aae, server=bd6592c37bb2,40087,1732486423713 2024-11-24T22:14:26,007 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732486460633.63842f73863e9e7b9587486b122e6aae., hostname=bd6592c37bb2,40087,1732486423713, seqNum=5 , the old value is region=TestAcidGuarantees,,1732486460633.63842f73863e9e7b9587486b122e6aae., hostname=bd6592c37bb2,40087,1732486423713, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=63842f73863e9e7b9587486b122e6aae, server=bd6592c37bb2,40087,1732486423713 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T22:14:26,007 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732486460633.63842f73863e9e7b9587486b122e6aae., hostname=bd6592c37bb2,40087,1732486423713, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=63842f73863e9e7b9587486b122e6aae, server=bd6592c37bb2,40087,1732486423713 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T22:14:26,007 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732486460633.63842f73863e9e7b9587486b122e6aae., hostname=bd6592c37bb2,40087,1732486423713, seqNum=5 , the old value is region=TestAcidGuarantees,,1732486460633.63842f73863e9e7b9587486b122e6aae., hostname=bd6592c37bb2,40087,1732486423713, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=63842f73863e9e7b9587486b122e6aae, server=bd6592c37bb2,40087,1732486423713 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T22:14:26,007 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732486460633.63842f73863e9e7b9587486b122e6aae., hostname=bd6592c37bb2,40087,1732486423713, seqNum=5 because the exception is null or not the one we care about 2024-11-24T22:14:26,007 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732486460633.63842f73863e9e7b9587486b122e6aae., hostname=bd6592c37bb2,40087,1732486423713, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=63842f73863e9e7b9587486b122e6aae, server=bd6592c37bb2,40087,1732486423713 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T22:14:26,007 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732486460633.63842f73863e9e7b9587486b122e6aae., hostname=bd6592c37bb2,40087,1732486423713, seqNum=5 because the exception is null or not the one we care about 2024-11-24T22:14:26,007 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40087 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=63842f73863e9e7b9587486b122e6aae, server=bd6592c37bb2,40087,1732486423713 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-24T22:14:26,007 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40087 {}] ipc.CallRunner(138): callId: 19 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40244 deadline: 1732486526001, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=63842f73863e9e7b9587486b122e6aae, server=bd6592c37bb2,40087,1732486423713 2024-11-24T22:14:26,008 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732486460633.63842f73863e9e7b9587486b122e6aae., hostname=bd6592c37bb2,40087,1732486423713, seqNum=5 , the old value is region=TestAcidGuarantees,,1732486460633.63842f73863e9e7b9587486b122e6aae., hostname=bd6592c37bb2,40087,1732486423713, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=63842f73863e9e7b9587486b122e6aae, server=bd6592c37bb2,40087,1732486423713 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T22:14:26,008 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732486460633.63842f73863e9e7b9587486b122e6aae., hostname=bd6592c37bb2,40087,1732486423713, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=63842f73863e9e7b9587486b122e6aae, server=bd6592c37bb2,40087,1732486423713 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T22:14:26,008 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732486460633.63842f73863e9e7b9587486b122e6aae., hostname=bd6592c37bb2,40087,1732486423713, seqNum=5 because the exception is null or not the one we care about 2024-11-24T22:14:26,071 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37701 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=40 2024-11-24T22:14:26,122 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=40087 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=41 2024-11-24T22:14:26,122 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bd6592c37bb2:0-1 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732486460633.63842f73863e9e7b9587486b122e6aae. 2024-11-24T22:14:26,122 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bd6592c37bb2:0-1 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732486460633.63842f73863e9e7b9587486b122e6aae. as already flushing 2024-11-24T22:14:26,122 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bd6592c37bb2:0-1 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732486460633.63842f73863e9e7b9587486b122e6aae. 2024-11-24T22:14:26,122 ERROR [RS_FLUSH_OPERATIONS-regionserver/bd6592c37bb2:0-1 {event_type=RS_FLUSH_REGIONS, pid=41}] handler.RSProcedureHandler(58): pid=41 java.io.IOException: Unable to complete flush {ENCODED => 63842f73863e9e7b9587486b122e6aae, NAME => 'TestAcidGuarantees,,1732486460633.63842f73863e9e7b9587486b122e6aae.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T22:14:26,122 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bd6592c37bb2:0-1 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=41 java.io.IOException: Unable to complete flush {ENCODED => 63842f73863e9e7b9587486b122e6aae, NAME => 'TestAcidGuarantees,,1732486460633.63842f73863e9e7b9587486b122e6aae.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T22:14:26,129 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37701 {}] master.HMaster(4177): Remote procedure failed, pid=41 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 63842f73863e9e7b9587486b122e6aae, NAME => 'TestAcidGuarantees,,1732486460633.63842f73863e9e7b9587486b122e6aae.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 63842f73863e9e7b9587486b122e6aae, NAME => 'TestAcidGuarantees,,1732486460633.63842f73863e9e7b9587486b122e6aae.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T22:14:26,192 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40087 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=63842f73863e9e7b9587486b122e6aae, server=bd6592c37bb2,40087,1732486423713 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-24T22:14:26,192 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40087 {}] ipc.CallRunner(138): callId: 26 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40202 deadline: 1732486526191, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=63842f73863e9e7b9587486b122e6aae, server=bd6592c37bb2,40087,1732486423713 2024-11-24T22:14:26,193 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732486460633.63842f73863e9e7b9587486b122e6aae., hostname=bd6592c37bb2,40087,1732486423713, seqNum=5 , the old value is region=TestAcidGuarantees,,1732486460633.63842f73863e9e7b9587486b122e6aae., hostname=bd6592c37bb2,40087,1732486423713, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=63842f73863e9e7b9587486b122e6aae, server=bd6592c37bb2,40087,1732486423713 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T22:14:26,193 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732486460633.63842f73863e9e7b9587486b122e6aae., hostname=bd6592c37bb2,40087,1732486423713, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=63842f73863e9e7b9587486b122e6aae, server=bd6592c37bb2,40087,1732486423713 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T22:14:26,193 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732486460633.63842f73863e9e7b9587486b122e6aae., hostname=bd6592c37bb2,40087,1732486423713, seqNum=5 because the exception is null or not the one we care about 2024-11-24T22:14:26,216 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40087 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=63842f73863e9e7b9587486b122e6aae, server=bd6592c37bb2,40087,1732486423713 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-24T22:14:26,216 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40087 {}] ipc.CallRunner(138): callId: 23 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40226 deadline: 1732486526211, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=63842f73863e9e7b9587486b122e6aae, server=bd6592c37bb2,40087,1732486423713 2024-11-24T22:14:26,217 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732486460633.63842f73863e9e7b9587486b122e6aae., hostname=bd6592c37bb2,40087,1732486423713, seqNum=5 , the old value is region=TestAcidGuarantees,,1732486460633.63842f73863e9e7b9587486b122e6aae., hostname=bd6592c37bb2,40087,1732486423713, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=63842f73863e9e7b9587486b122e6aae, server=bd6592c37bb2,40087,1732486423713 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T22:14:26,217 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732486460633.63842f73863e9e7b9587486b122e6aae., hostname=bd6592c37bb2,40087,1732486423713, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=63842f73863e9e7b9587486b122e6aae, server=bd6592c37bb2,40087,1732486423713 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T22:14:26,217 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732486460633.63842f73863e9e7b9587486b122e6aae., hostname=bd6592c37bb2,40087,1732486423713, seqNum=5 because the exception is null or not the one we care about 2024-11-24T22:14:26,217 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40087 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=63842f73863e9e7b9587486b122e6aae, server=bd6592c37bb2,40087,1732486423713 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-24T22:14:26,217 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40087 {}] ipc.CallRunner(138): callId: 22 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40232 deadline: 1732486526211, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=63842f73863e9e7b9587486b122e6aae, server=bd6592c37bb2,40087,1732486423713 2024-11-24T22:14:26,218 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40087 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=63842f73863e9e7b9587486b122e6aae, server=bd6592c37bb2,40087,1732486423713 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-24T22:14:26,218 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732486460633.63842f73863e9e7b9587486b122e6aae., hostname=bd6592c37bb2,40087,1732486423713, seqNum=5 , the old value is region=TestAcidGuarantees,,1732486460633.63842f73863e9e7b9587486b122e6aae., hostname=bd6592c37bb2,40087,1732486423713, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=63842f73863e9e7b9587486b122e6aae, server=bd6592c37bb2,40087,1732486423713 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T22:14:26,218 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732486460633.63842f73863e9e7b9587486b122e6aae., hostname=bd6592c37bb2,40087,1732486423713, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=63842f73863e9e7b9587486b122e6aae, server=bd6592c37bb2,40087,1732486423713 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T22:14:26,218 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40087 {}] ipc.CallRunner(138): callId: 26 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40206 deadline: 1732486526211, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=63842f73863e9e7b9587486b122e6aae, server=bd6592c37bb2,40087,1732486423713 2024-11-24T22:14:26,218 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732486460633.63842f73863e9e7b9587486b122e6aae., hostname=bd6592c37bb2,40087,1732486423713, seqNum=5 because the exception is null or not the one we care about 2024-11-24T22:14:26,218 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40087 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=63842f73863e9e7b9587486b122e6aae, server=bd6592c37bb2,40087,1732486423713 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-24T22:14:26,219 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732486460633.63842f73863e9e7b9587486b122e6aae., hostname=bd6592c37bb2,40087,1732486423713, seqNum=5 , the old value is region=TestAcidGuarantees,,1732486460633.63842f73863e9e7b9587486b122e6aae., hostname=bd6592c37bb2,40087,1732486423713, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=63842f73863e9e7b9587486b122e6aae, server=bd6592c37bb2,40087,1732486423713 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T22:14:26,219 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40087 {}] ipc.CallRunner(138): callId: 20 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40244 deadline: 1732486526214, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=63842f73863e9e7b9587486b122e6aae, server=bd6592c37bb2,40087,1732486423713 2024-11-24T22:14:26,219 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732486460633.63842f73863e9e7b9587486b122e6aae., hostname=bd6592c37bb2,40087,1732486423713, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=63842f73863e9e7b9587486b122e6aae, server=bd6592c37bb2,40087,1732486423713 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T22:14:26,219 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732486460633.63842f73863e9e7b9587486b122e6aae., hostname=bd6592c37bb2,40087,1732486423713, seqNum=5 because the exception is null or not the one we care about 2024-11-24T22:14:26,219 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732486460633.63842f73863e9e7b9587486b122e6aae., hostname=bd6592c37bb2,40087,1732486423713, seqNum=5 , the old value is region=TestAcidGuarantees,,1732486460633.63842f73863e9e7b9587486b122e6aae., hostname=bd6592c37bb2,40087,1732486423713, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=63842f73863e9e7b9587486b122e6aae, server=bd6592c37bb2,40087,1732486423713 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T22:14:26,219 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732486460633.63842f73863e9e7b9587486b122e6aae., hostname=bd6592c37bb2,40087,1732486423713, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=63842f73863e9e7b9587486b122e6aae, server=bd6592c37bb2,40087,1732486423713 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T22:14:26,219 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732486460633.63842f73863e9e7b9587486b122e6aae., hostname=bd6592c37bb2,40087,1732486423713, seqNum=5 because the exception is null or not the one we care about 2024-11-24T22:14:26,281 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37701 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=40 2024-11-24T22:14:26,282 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=40087 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=41 2024-11-24T22:14:26,282 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bd6592c37bb2:0-2 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732486460633.63842f73863e9e7b9587486b122e6aae. 2024-11-24T22:14:26,282 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bd6592c37bb2:0-2 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732486460633.63842f73863e9e7b9587486b122e6aae. as already flushing 2024-11-24T22:14:26,282 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bd6592c37bb2:0-2 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732486460633.63842f73863e9e7b9587486b122e6aae. 2024-11-24T22:14:26,282 ERROR [RS_FLUSH_OPERATIONS-regionserver/bd6592c37bb2:0-2 {event_type=RS_FLUSH_REGIONS, pid=41}] handler.RSProcedureHandler(58): pid=41 java.io.IOException: Unable to complete flush {ENCODED => 63842f73863e9e7b9587486b122e6aae, NAME => 'TestAcidGuarantees,,1732486460633.63842f73863e9e7b9587486b122e6aae.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T22:14:26,282 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bd6592c37bb2:0-2 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=41 java.io.IOException: Unable to complete flush {ENCODED => 63842f73863e9e7b9587486b122e6aae, NAME => 'TestAcidGuarantees,,1732486460633.63842f73863e9e7b9587486b122e6aae.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T22:14:26,284 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37701 {}] master.HMaster(4177): Remote procedure failed, pid=41 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 63842f73863e9e7b9587486b122e6aae, NAME => 'TestAcidGuarantees,,1732486460633.63842f73863e9e7b9587486b122e6aae.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 63842f73863e9e7b9587486b122e6aae, NAME => 'TestAcidGuarantees,,1732486460633.63842f73863e9e7b9587486b122e6aae.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T22:14:26,313 INFO [master/bd6592c37bb2:0.Chore.1 {}] migrate.RollingUpgradeChore(116): There is no table to migrate StoreFileTracker! 2024-11-24T22:14:26,313 INFO [master/bd6592c37bb2:0.Chore.1 {}] migrate.RollingUpgradeChore(85): All Rolling-Upgrade tasks are complete, shutdown RollingUpgradeChore! 2024-11-24T22:14:26,333 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=65, memsize=24.6 K, hasBloomFilter=true, into tmp file hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/63842f73863e9e7b9587486b122e6aae/.tmp/A/58aff1a92da2493ab1bf579d455a3632 2024-11-24T22:14:26,363 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/63842f73863e9e7b9587486b122e6aae/.tmp/B/026af172cb4249039e2f443f841ab798 is 50, key is test_row_0/B:col10/1732486465806/Put/seqid=0 2024-11-24T22:14:26,420 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45929 is added to blk_1073741949_1125 (size=12001) 2024-11-24T22:14:26,422 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=24.60 KB at sequenceid=65 (bloomFilter=true), to=hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/63842f73863e9e7b9587486b122e6aae/.tmp/B/026af172cb4249039e2f443f841ab798 2024-11-24T22:14:26,438 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=40087 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=41 2024-11-24T22:14:26,441 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bd6592c37bb2:0-0 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732486460633.63842f73863e9e7b9587486b122e6aae. 2024-11-24T22:14:26,441 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bd6592c37bb2:0-0 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732486460633.63842f73863e9e7b9587486b122e6aae. as already flushing 2024-11-24T22:14:26,441 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bd6592c37bb2:0-0 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732486460633.63842f73863e9e7b9587486b122e6aae. 2024-11-24T22:14:26,441 ERROR [RS_FLUSH_OPERATIONS-regionserver/bd6592c37bb2:0-0 {event_type=RS_FLUSH_REGIONS, pid=41}] handler.RSProcedureHandler(58): pid=41 java.io.IOException: Unable to complete flush {ENCODED => 63842f73863e9e7b9587486b122e6aae, NAME => 'TestAcidGuarantees,,1732486460633.63842f73863e9e7b9587486b122e6aae.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T22:14:26,441 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bd6592c37bb2:0-0 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=41 java.io.IOException: Unable to complete flush {ENCODED => 63842f73863e9e7b9587486b122e6aae, NAME => 'TestAcidGuarantees,,1732486460633.63842f73863e9e7b9587486b122e6aae.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T22:14:26,442 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37701 {}] master.HMaster(4177): Remote procedure failed, pid=41 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 63842f73863e9e7b9587486b122e6aae, NAME => 'TestAcidGuarantees,,1732486460633.63842f73863e9e7b9587486b122e6aae.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 63842f73863e9e7b9587486b122e6aae, NAME => 'TestAcidGuarantees,,1732486460633.63842f73863e9e7b9587486b122e6aae.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T22:14:26,451 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/63842f73863e9e7b9587486b122e6aae/.tmp/C/2e9f8709e5ad4ad5a5f86d4aa09c9c61 is 50, key is test_row_0/C:col10/1732486465806/Put/seqid=0 2024-11-24T22:14:26,496 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45929 is added to blk_1073741950_1126 (size=12001) 2024-11-24T22:14:26,497 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=24.60 KB at sequenceid=65 (bloomFilter=true), to=hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/63842f73863e9e7b9587486b122e6aae/.tmp/C/2e9f8709e5ad4ad5a5f86d4aa09c9c61 2024-11-24T22:14:26,502 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40087 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=63842f73863e9e7b9587486b122e6aae, server=bd6592c37bb2,40087,1732486423713 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-24T22:14:26,503 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40087 {}] ipc.CallRunner(138): callId: 27 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40202 deadline: 1732486526501, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=63842f73863e9e7b9587486b122e6aae, server=bd6592c37bb2,40087,1732486423713 2024-11-24T22:14:26,504 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732486460633.63842f73863e9e7b9587486b122e6aae., hostname=bd6592c37bb2,40087,1732486423713, seqNum=5 , the old value is region=TestAcidGuarantees,,1732486460633.63842f73863e9e7b9587486b122e6aae., hostname=bd6592c37bb2,40087,1732486423713, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=63842f73863e9e7b9587486b122e6aae, server=bd6592c37bb2,40087,1732486423713 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T22:14:26,504 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732486460633.63842f73863e9e7b9587486b122e6aae., hostname=bd6592c37bb2,40087,1732486423713, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=63842f73863e9e7b9587486b122e6aae, server=bd6592c37bb2,40087,1732486423713 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T22:14:26,504 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732486460633.63842f73863e9e7b9587486b122e6aae., hostname=bd6592c37bb2,40087,1732486423713, seqNum=5 because the exception is null or not the one we care about 2024-11-24T22:14:26,504 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/63842f73863e9e7b9587486b122e6aae/.tmp/A/58aff1a92da2493ab1bf579d455a3632 as hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/63842f73863e9e7b9587486b122e6aae/A/58aff1a92da2493ab1bf579d455a3632 2024-11-24T22:14:26,511 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/63842f73863e9e7b9587486b122e6aae/A/58aff1a92da2493ab1bf579d455a3632, entries=250, sequenceid=65, filesize=47.0 K 2024-11-24T22:14:26,512 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/63842f73863e9e7b9587486b122e6aae/.tmp/B/026af172cb4249039e2f443f841ab798 as hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/63842f73863e9e7b9587486b122e6aae/B/026af172cb4249039e2f443f841ab798 2024-11-24T22:14:26,521 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/63842f73863e9e7b9587486b122e6aae/B/026af172cb4249039e2f443f841ab798, entries=150, sequenceid=65, filesize=11.7 K 2024-11-24T22:14:26,522 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40087 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=63842f73863e9e7b9587486b122e6aae, server=bd6592c37bb2,40087,1732486423713 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-24T22:14:26,522 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40087 {}] ipc.CallRunner(138): callId: 24 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40226 deadline: 1732486526521, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=63842f73863e9e7b9587486b122e6aae, server=bd6592c37bb2,40087,1732486423713 2024-11-24T22:14:26,522 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/63842f73863e9e7b9587486b122e6aae/.tmp/C/2e9f8709e5ad4ad5a5f86d4aa09c9c61 as hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/63842f73863e9e7b9587486b122e6aae/C/2e9f8709e5ad4ad5a5f86d4aa09c9c61 2024-11-24T22:14:26,523 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40087 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=63842f73863e9e7b9587486b122e6aae, server=bd6592c37bb2,40087,1732486423713 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-24T22:14:26,523 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732486460633.63842f73863e9e7b9587486b122e6aae., hostname=bd6592c37bb2,40087,1732486423713, seqNum=5 , the old value is region=TestAcidGuarantees,,1732486460633.63842f73863e9e7b9587486b122e6aae., hostname=bd6592c37bb2,40087,1732486423713, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=63842f73863e9e7b9587486b122e6aae, server=bd6592c37bb2,40087,1732486423713 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T22:14:26,523 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732486460633.63842f73863e9e7b9587486b122e6aae., hostname=bd6592c37bb2,40087,1732486423713, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=63842f73863e9e7b9587486b122e6aae, server=bd6592c37bb2,40087,1732486423713 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T22:14:26,523 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732486460633.63842f73863e9e7b9587486b122e6aae., hostname=bd6592c37bb2,40087,1732486423713, seqNum=5 because the exception is null or not the one we care about 2024-11-24T22:14:26,523 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40087 {}] ipc.CallRunner(138): callId: 23 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40232 deadline: 1732486526521, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=63842f73863e9e7b9587486b122e6aae, server=bd6592c37bb2,40087,1732486423713 2024-11-24T22:14:26,523 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732486460633.63842f73863e9e7b9587486b122e6aae., hostname=bd6592c37bb2,40087,1732486423713, seqNum=5 , the old value is region=TestAcidGuarantees,,1732486460633.63842f73863e9e7b9587486b122e6aae., hostname=bd6592c37bb2,40087,1732486423713, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=63842f73863e9e7b9587486b122e6aae, server=bd6592c37bb2,40087,1732486423713 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T22:14:26,524 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732486460633.63842f73863e9e7b9587486b122e6aae., hostname=bd6592c37bb2,40087,1732486423713, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=63842f73863e9e7b9587486b122e6aae, server=bd6592c37bb2,40087,1732486423713 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T22:14:26,524 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732486460633.63842f73863e9e7b9587486b122e6aae., hostname=bd6592c37bb2,40087,1732486423713, seqNum=5 because the exception is null or not the one we care about 2024-11-24T22:14:26,528 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/63842f73863e9e7b9587486b122e6aae/C/2e9f8709e5ad4ad5a5f86d4aa09c9c61, entries=150, sequenceid=65, filesize=11.7 K 2024-11-24T22:14:26,530 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~73.80 KB/75570, heapSize ~219.84 KB/225120, currentSize=100.63 KB/103050 for 63842f73863e9e7b9587486b122e6aae in 721ms, sequenceid=65, compaction requested=true 2024-11-24T22:14:26,530 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 63842f73863e9e7b9587486b122e6aae: 2024-11-24T22:14:26,530 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 63842f73863e9e7b9587486b122e6aae:A, priority=-2147483648, current under compaction store size is 1 2024-11-24T22:14:26,530 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-24T22:14:26,530 DEBUG [RS:0;bd6592c37bb2:40087-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-24T22:14:26,530 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 63842f73863e9e7b9587486b122e6aae:B, priority=-2147483648, current under compaction store size is 2 2024-11-24T22:14:26,530 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-24T22:14:26,530 DEBUG [RS:0;bd6592c37bb2:40087-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-24T22:14:26,530 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 63842f73863e9e7b9587486b122e6aae:C, priority=-2147483648, current under compaction store size is 3 2024-11-24T22:14:26,530 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-24T22:14:26,532 DEBUG [RS:0;bd6592c37bb2:40087-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 135827 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-24T22:14:26,532 DEBUG [RS:0;bd6592c37bb2:40087-shortCompactions-0 {}] regionserver.HStore(1541): 63842f73863e9e7b9587486b122e6aae/A is initiating minor compaction (all files) 2024-11-24T22:14:26,533 INFO [RS:0;bd6592c37bb2:40087-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 63842f73863e9e7b9587486b122e6aae/A in TestAcidGuarantees,,1732486460633.63842f73863e9e7b9587486b122e6aae. 2024-11-24T22:14:26,533 INFO [RS:0;bd6592c37bb2:40087-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/63842f73863e9e7b9587486b122e6aae/A/13672022da204af48915a725fbfa98a5, hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/63842f73863e9e7b9587486b122e6aae/A/fade54e87e5f4b1b93f1630813554439, hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/63842f73863e9e7b9587486b122e6aae/A/58aff1a92da2493ab1bf579d455a3632] into tmpdir=hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/63842f73863e9e7b9587486b122e6aae/.tmp, totalSize=132.6 K 2024-11-24T22:14:26,533 INFO [RS:0;bd6592c37bb2:40087-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=13 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1732486460633.63842f73863e9e7b9587486b122e6aae. 2024-11-24T22:14:26,533 DEBUG [RS:0;bd6592c37bb2:40087-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1732486460633.63842f73863e9e7b9587486b122e6aae. files: [hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/63842f73863e9e7b9587486b122e6aae/A/13672022da204af48915a725fbfa98a5, hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/63842f73863e9e7b9587486b122e6aae/A/fade54e87e5f4b1b93f1630813554439, hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/63842f73863e9e7b9587486b122e6aae/A/58aff1a92da2493ab1bf579d455a3632] 2024-11-24T22:14:26,535 DEBUG [RS:0;bd6592c37bb2:40087-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36003 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-24T22:14:26,536 DEBUG [RS:0;bd6592c37bb2:40087-longCompactions-0 {}] regionserver.HStore(1541): 63842f73863e9e7b9587486b122e6aae/B is initiating minor compaction (all files) 2024-11-24T22:14:26,536 INFO [RS:0;bd6592c37bb2:40087-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 63842f73863e9e7b9587486b122e6aae/B in TestAcidGuarantees,,1732486460633.63842f73863e9e7b9587486b122e6aae. 2024-11-24T22:14:26,536 INFO [RS:0;bd6592c37bb2:40087-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/63842f73863e9e7b9587486b122e6aae/B/e0d8bb8944b64909b312bea8b1cacdd8, hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/63842f73863e9e7b9587486b122e6aae/B/02bd6ba0eecd48b99e5ad15a050a2011, hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/63842f73863e9e7b9587486b122e6aae/B/026af172cb4249039e2f443f841ab798] into tmpdir=hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/63842f73863e9e7b9587486b122e6aae/.tmp, totalSize=35.2 K 2024-11-24T22:14:26,537 DEBUG [RS:0;bd6592c37bb2:40087-shortCompactions-0 {}] compactions.Compactor(225): Compacting 13672022da204af48915a725fbfa98a5, keycount=300, bloomtype=ROW, size=55.4 K, encoding=NONE, compression=NONE, seqNum=15, earliestPutTs=1732486463836 2024-11-24T22:14:26,537 DEBUG [RS:0;bd6592c37bb2:40087-longCompactions-0 {}] compactions.Compactor(225): Compacting e0d8bb8944b64909b312bea8b1cacdd8, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=15, earliestPutTs=1732486463836 2024-11-24T22:14:26,538 DEBUG [RS:0;bd6592c37bb2:40087-shortCompactions-0 {}] compactions.Compactor(225): Compacting fade54e87e5f4b1b93f1630813554439, keycount=150, bloomtype=ROW, size=30.2 K, encoding=NONE, compression=NONE, seqNum=51, earliestPutTs=1732486463969 2024-11-24T22:14:26,538 DEBUG [RS:0;bd6592c37bb2:40087-longCompactions-0 {}] compactions.Compactor(225): Compacting 02bd6ba0eecd48b99e5ad15a050a2011, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=51, earliestPutTs=1732486463969 2024-11-24T22:14:26,539 DEBUG [RS:0;bd6592c37bb2:40087-shortCompactions-0 {}] compactions.Compactor(225): Compacting 58aff1a92da2493ab1bf579d455a3632, keycount=250, bloomtype=ROW, size=47.0 K, encoding=NONE, compression=NONE, seqNum=65, earliestPutTs=1732486465173 2024-11-24T22:14:26,541 DEBUG [RS:0;bd6592c37bb2:40087-longCompactions-0 {}] compactions.Compactor(225): Compacting 026af172cb4249039e2f443f841ab798, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=65, earliestPutTs=1732486465173 2024-11-24T22:14:26,545 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40087 {}] regionserver.HRegion(8855): Flush requested on 63842f73863e9e7b9587486b122e6aae 2024-11-24T22:14:26,545 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 63842f73863e9e7b9587486b122e6aae 3/3 column families, dataSize=114.05 KB heapSize=339.42 KB 2024-11-24T22:14:26,545 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 63842f73863e9e7b9587486b122e6aae, store=A 2024-11-24T22:14:26,545 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-24T22:14:26,545 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 63842f73863e9e7b9587486b122e6aae, store=B 2024-11-24T22:14:26,545 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-24T22:14:26,545 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 63842f73863e9e7b9587486b122e6aae, store=C 2024-11-24T22:14:26,545 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-24T22:14:26,573 INFO [RS:0;bd6592c37bb2:40087-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=63842f73863e9e7b9587486b122e6aae] 2024-11-24T22:14:26,591 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37701 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=40 2024-11-24T22:14:26,595 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=40087 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=41 2024-11-24T22:14:26,595 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bd6592c37bb2:0-1 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732486460633.63842f73863e9e7b9587486b122e6aae. 2024-11-24T22:14:26,595 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bd6592c37bb2:0-1 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732486460633.63842f73863e9e7b9587486b122e6aae. as already flushing 2024-11-24T22:14:26,595 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bd6592c37bb2:0-1 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732486460633.63842f73863e9e7b9587486b122e6aae. 2024-11-24T22:14:26,595 INFO [RS:0;bd6592c37bb2:40087-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 63842f73863e9e7b9587486b122e6aae#B#compaction#109 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-24T22:14:26,595 ERROR [RS_FLUSH_OPERATIONS-regionserver/bd6592c37bb2:0-1 {event_type=RS_FLUSH_REGIONS, pid=41}] handler.RSProcedureHandler(58): pid=41 java.io.IOException: Unable to complete flush {ENCODED => 63842f73863e9e7b9587486b122e6aae, NAME => 'TestAcidGuarantees,,1732486460633.63842f73863e9e7b9587486b122e6aae.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T22:14:26,595 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bd6592c37bb2:0-1 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=41 java.io.IOException: Unable to complete flush {ENCODED => 63842f73863e9e7b9587486b122e6aae, NAME => 'TestAcidGuarantees,,1732486460633.63842f73863e9e7b9587486b122e6aae.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T22:14:26,596 DEBUG [RS:0;bd6592c37bb2:40087-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/63842f73863e9e7b9587486b122e6aae/.tmp/B/b4285931220446c38584c66287d3a5b9 is 50, key is test_row_0/B:col10/1732486465806/Put/seqid=0 2024-11-24T22:14:26,596 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37701 {}] master.HMaster(4177): Remote procedure failed, pid=41 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 63842f73863e9e7b9587486b122e6aae, NAME => 'TestAcidGuarantees,,1732486460633.63842f73863e9e7b9587486b122e6aae.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 63842f73863e9e7b9587486b122e6aae, NAME => 'TestAcidGuarantees,,1732486460633.63842f73863e9e7b9587486b122e6aae.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T22:14:26,602 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40087 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=63842f73863e9e7b9587486b122e6aae, server=bd6592c37bb2,40087,1732486423713 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-24T22:14:26,603 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40087 {}] ipc.CallRunner(138): callId: 26 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40244 deadline: 1732486526598, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=63842f73863e9e7b9587486b122e6aae, server=bd6592c37bb2,40087,1732486423713 2024-11-24T22:14:26,605 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732486460633.63842f73863e9e7b9587486b122e6aae., hostname=bd6592c37bb2,40087,1732486423713, seqNum=5 , the old value is region=TestAcidGuarantees,,1732486460633.63842f73863e9e7b9587486b122e6aae., hostname=bd6592c37bb2,40087,1732486423713, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=63842f73863e9e7b9587486b122e6aae, server=bd6592c37bb2,40087,1732486423713 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T22:14:26,605 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732486460633.63842f73863e9e7b9587486b122e6aae., hostname=bd6592c37bb2,40087,1732486423713, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=63842f73863e9e7b9587486b122e6aae, server=bd6592c37bb2,40087,1732486423713 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T22:14:26,605 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732486460633.63842f73863e9e7b9587486b122e6aae., hostname=bd6592c37bb2,40087,1732486423713, seqNum=5 because the exception is null or not the one we care about 2024-11-24T22:14:26,606 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241124903c41653ee343e4bdfa839688befd68_63842f73863e9e7b9587486b122e6aae is 50, key is test_row_0/A:col10/1732486465856/Put/seqid=0 2024-11-24T22:14:26,607 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40087 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=63842f73863e9e7b9587486b122e6aae, server=bd6592c37bb2,40087,1732486423713 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-24T22:14:26,607 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40087 {}] ipc.CallRunner(138): callId: 33 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40206 deadline: 1732486526606, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=63842f73863e9e7b9587486b122e6aae, server=bd6592c37bb2,40087,1732486423713 2024-11-24T22:14:26,608 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732486460633.63842f73863e9e7b9587486b122e6aae., hostname=bd6592c37bb2,40087,1732486423713, seqNum=5 , the old value is region=TestAcidGuarantees,,1732486460633.63842f73863e9e7b9587486b122e6aae., hostname=bd6592c37bb2,40087,1732486423713, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=63842f73863e9e7b9587486b122e6aae, server=bd6592c37bb2,40087,1732486423713 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T22:14:26,608 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732486460633.63842f73863e9e7b9587486b122e6aae., hostname=bd6592c37bb2,40087,1732486423713, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=63842f73863e9e7b9587486b122e6aae, server=bd6592c37bb2,40087,1732486423713 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T22:14:26,608 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732486460633.63842f73863e9e7b9587486b122e6aae., hostname=bd6592c37bb2,40087,1732486423713, seqNum=5 because the exception is null or not the one we care about 2024-11-24T22:14:26,612 DEBUG [RS:0;bd6592c37bb2:40087-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e20241124493b751c06cb42cfa12a374e5662bb0a_63842f73863e9e7b9587486b122e6aae store=[table=TestAcidGuarantees family=A region=63842f73863e9e7b9587486b122e6aae] 2024-11-24T22:14:26,617 DEBUG [RS:0;bd6592c37bb2:40087-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e20241124493b751c06cb42cfa12a374e5662bb0a_63842f73863e9e7b9587486b122e6aae, store=[table=TestAcidGuarantees family=A region=63842f73863e9e7b9587486b122e6aae] 2024-11-24T22:14:26,618 DEBUG [RS:0;bd6592c37bb2:40087-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241124493b751c06cb42cfa12a374e5662bb0a_63842f73863e9e7b9587486b122e6aae because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=63842f73863e9e7b9587486b122e6aae] 2024-11-24T22:14:26,641 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45929 is added to blk_1073741951_1127 (size=12104) 2024-11-24T22:14:26,650 DEBUG [RS:0;bd6592c37bb2:40087-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/63842f73863e9e7b9587486b122e6aae/.tmp/B/b4285931220446c38584c66287d3a5b9 as hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/63842f73863e9e7b9587486b122e6aae/B/b4285931220446c38584c66287d3a5b9 2024-11-24T22:14:26,656 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45929 is added to blk_1073741952_1128 (size=12154) 2024-11-24T22:14:26,657 INFO [RS:0;bd6592c37bb2:40087-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 63842f73863e9e7b9587486b122e6aae/B of 63842f73863e9e7b9587486b122e6aae into b4285931220446c38584c66287d3a5b9(size=11.8 K), total size for store is 11.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-24T22:14:26,657 DEBUG [RS:0;bd6592c37bb2:40087-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 63842f73863e9e7b9587486b122e6aae: 2024-11-24T22:14:26,658 INFO [RS:0;bd6592c37bb2:40087-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732486460633.63842f73863e9e7b9587486b122e6aae., storeName=63842f73863e9e7b9587486b122e6aae/B, priority=13, startTime=1732486466530; duration=0sec 2024-11-24T22:14:26,658 DEBUG [RS:0;bd6592c37bb2:40087-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-24T22:14:26,658 DEBUG [RS:0;bd6592c37bb2:40087-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 63842f73863e9e7b9587486b122e6aae:B 2024-11-24T22:14:26,658 DEBUG [RS:0;bd6592c37bb2:40087-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-24T22:14:26,659 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T22:14:26,660 DEBUG [RS:0;bd6592c37bb2:40087-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36003 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-24T22:14:26,660 DEBUG [RS:0;bd6592c37bb2:40087-longCompactions-0 {}] regionserver.HStore(1541): 63842f73863e9e7b9587486b122e6aae/C is initiating minor compaction (all files) 2024-11-24T22:14:26,661 INFO [RS:0;bd6592c37bb2:40087-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 63842f73863e9e7b9587486b122e6aae/C in TestAcidGuarantees,,1732486460633.63842f73863e9e7b9587486b122e6aae. 2024-11-24T22:14:26,661 INFO [RS:0;bd6592c37bb2:40087-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/63842f73863e9e7b9587486b122e6aae/C/6711daa1d8ff424e8ac2f9cd4c7287f4, hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/63842f73863e9e7b9587486b122e6aae/C/f967706d7c7a4b2aa41d0ce3d33ee73e, hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/63842f73863e9e7b9587486b122e6aae/C/2e9f8709e5ad4ad5a5f86d4aa09c9c61] into tmpdir=hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/63842f73863e9e7b9587486b122e6aae/.tmp, totalSize=35.2 K 2024-11-24T22:14:26,663 DEBUG [RS:0;bd6592c37bb2:40087-longCompactions-0 {}] compactions.Compactor(225): Compacting 6711daa1d8ff424e8ac2f9cd4c7287f4, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=15, earliestPutTs=1732486463836 2024-11-24T22:14:26,665 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241124903c41653ee343e4bdfa839688befd68_63842f73863e9e7b9587486b122e6aae to hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241124903c41653ee343e4bdfa839688befd68_63842f73863e9e7b9587486b122e6aae 2024-11-24T22:14:26,665 DEBUG [RS:0;bd6592c37bb2:40087-longCompactions-0 {}] compactions.Compactor(225): Compacting f967706d7c7a4b2aa41d0ce3d33ee73e, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=51, earliestPutTs=1732486463969 2024-11-24T22:14:26,666 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/63842f73863e9e7b9587486b122e6aae/.tmp/A/f00867b3407d44fa9ab08e3464c78907, store: [table=TestAcidGuarantees family=A region=63842f73863e9e7b9587486b122e6aae] 2024-11-24T22:14:26,666 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/63842f73863e9e7b9587486b122e6aae/.tmp/A/f00867b3407d44fa9ab08e3464c78907 is 175, key is test_row_0/A:col10/1732486465856/Put/seqid=0 2024-11-24T22:14:26,667 DEBUG [RS:0;bd6592c37bb2:40087-longCompactions-0 {}] compactions.Compactor(225): Compacting 2e9f8709e5ad4ad5a5f86d4aa09c9c61, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=65, earliestPutTs=1732486465173 2024-11-24T22:14:26,683 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45929 is added to blk_1073741953_1129 (size=4469) 2024-11-24T22:14:26,685 INFO [RS:0;bd6592c37bb2:40087-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 63842f73863e9e7b9587486b122e6aae#A#compaction#108 average throughput is 0.22 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-24T22:14:26,686 DEBUG [RS:0;bd6592c37bb2:40087-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/63842f73863e9e7b9587486b122e6aae/.tmp/A/61c5cebf766f4aa09183bcfd524ae066 is 175, key is test_row_0/A:col10/1732486465806/Put/seqid=0 2024-11-24T22:14:26,713 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40087 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=63842f73863e9e7b9587486b122e6aae, server=bd6592c37bb2,40087,1732486423713 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-24T22:14:26,714 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40087 {}] ipc.CallRunner(138): callId: 34 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40206 deadline: 1732486526711, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=63842f73863e9e7b9587486b122e6aae, server=bd6592c37bb2,40087,1732486423713 2024-11-24T22:14:26,714 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732486460633.63842f73863e9e7b9587486b122e6aae., hostname=bd6592c37bb2,40087,1732486423713, seqNum=5 , the old value is region=TestAcidGuarantees,,1732486460633.63842f73863e9e7b9587486b122e6aae., hostname=bd6592c37bb2,40087,1732486423713, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=63842f73863e9e7b9587486b122e6aae, server=bd6592c37bb2,40087,1732486423713 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T22:14:26,714 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732486460633.63842f73863e9e7b9587486b122e6aae., hostname=bd6592c37bb2,40087,1732486423713, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=63842f73863e9e7b9587486b122e6aae, server=bd6592c37bb2,40087,1732486423713 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T22:14:26,714 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732486460633.63842f73863e9e7b9587486b122e6aae., hostname=bd6592c37bb2,40087,1732486423713, seqNum=5 because the exception is null or not the one we care about 2024-11-24T22:14:26,719 INFO [RS:0;bd6592c37bb2:40087-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 63842f73863e9e7b9587486b122e6aae#C#compaction#111 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-24T22:14:26,720 DEBUG [RS:0;bd6592c37bb2:40087-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/63842f73863e9e7b9587486b122e6aae/.tmp/C/c3ebf3de6c6240cdb34d8d2cd189aadf is 50, key is test_row_0/C:col10/1732486465806/Put/seqid=0 2024-11-24T22:14:26,720 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40087 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=63842f73863e9e7b9587486b122e6aae, server=bd6592c37bb2,40087,1732486423713 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-24T22:14:26,720 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40087 {}] ipc.CallRunner(138): callId: 27 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40244 deadline: 1732486526712, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=63842f73863e9e7b9587486b122e6aae, server=bd6592c37bb2,40087,1732486423713 2024-11-24T22:14:26,721 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732486460633.63842f73863e9e7b9587486b122e6aae., hostname=bd6592c37bb2,40087,1732486423713, seqNum=5 , the old value is region=TestAcidGuarantees,,1732486460633.63842f73863e9e7b9587486b122e6aae., hostname=bd6592c37bb2,40087,1732486423713, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=63842f73863e9e7b9587486b122e6aae, server=bd6592c37bb2,40087,1732486423713 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T22:14:26,721 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732486460633.63842f73863e9e7b9587486b122e6aae., hostname=bd6592c37bb2,40087,1732486423713, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=63842f73863e9e7b9587486b122e6aae, server=bd6592c37bb2,40087,1732486423713 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T22:14:26,721 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732486460633.63842f73863e9e7b9587486b122e6aae., hostname=bd6592c37bb2,40087,1732486423713, seqNum=5 because the exception is null or not the one we care about 2024-11-24T22:14:26,749 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=40087 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=41 2024-11-24T22:14:26,749 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bd6592c37bb2:0-2 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732486460633.63842f73863e9e7b9587486b122e6aae. 2024-11-24T22:14:26,749 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bd6592c37bb2:0-2 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732486460633.63842f73863e9e7b9587486b122e6aae. as already flushing 2024-11-24T22:14:26,749 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bd6592c37bb2:0-2 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732486460633.63842f73863e9e7b9587486b122e6aae. 2024-11-24T22:14:26,749 ERROR [RS_FLUSH_OPERATIONS-regionserver/bd6592c37bb2:0-2 {event_type=RS_FLUSH_REGIONS, pid=41}] handler.RSProcedureHandler(58): pid=41 java.io.IOException: Unable to complete flush {ENCODED => 63842f73863e9e7b9587486b122e6aae, NAME => 'TestAcidGuarantees,,1732486460633.63842f73863e9e7b9587486b122e6aae.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T22:14:26,749 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bd6592c37bb2:0-2 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=41 java.io.IOException: Unable to complete flush {ENCODED => 63842f73863e9e7b9587486b122e6aae, NAME => 'TestAcidGuarantees,,1732486460633.63842f73863e9e7b9587486b122e6aae.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T22:14:26,751 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37701 {}] master.HMaster(4177): Remote procedure failed, pid=41 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 63842f73863e9e7b9587486b122e6aae, NAME => 'TestAcidGuarantees,,1732486460633.63842f73863e9e7b9587486b122e6aae.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 63842f73863e9e7b9587486b122e6aae, NAME => 'TestAcidGuarantees,,1732486460633.63842f73863e9e7b9587486b122e6aae.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T22:14:26,751 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45929 is added to blk_1073741954_1130 (size=30955) 2024-11-24T22:14:26,774 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45929 is added to blk_1073741955_1131 (size=31058) 2024-11-24T22:14:26,782 DEBUG [RS:0;bd6592c37bb2:40087-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/63842f73863e9e7b9587486b122e6aae/.tmp/A/61c5cebf766f4aa09183bcfd524ae066 as hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/63842f73863e9e7b9587486b122e6aae/A/61c5cebf766f4aa09183bcfd524ae066 2024-11-24T22:14:26,789 INFO [RS:0;bd6592c37bb2:40087-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 63842f73863e9e7b9587486b122e6aae/A of 63842f73863e9e7b9587486b122e6aae into 61c5cebf766f4aa09183bcfd524ae066(size=30.3 K), total size for store is 30.3 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-24T22:14:26,789 DEBUG [RS:0;bd6592c37bb2:40087-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 63842f73863e9e7b9587486b122e6aae: 2024-11-24T22:14:26,789 INFO [RS:0;bd6592c37bb2:40087-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732486460633.63842f73863e9e7b9587486b122e6aae., storeName=63842f73863e9e7b9587486b122e6aae/A, priority=13, startTime=1732486466530; duration=0sec 2024-11-24T22:14:26,789 DEBUG [RS:0;bd6592c37bb2:40087-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-24T22:14:26,789 DEBUG [RS:0;bd6592c37bb2:40087-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 63842f73863e9e7b9587486b122e6aae:A 2024-11-24T22:14:26,796 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45929 is added to blk_1073741956_1132 (size=12104) 2024-11-24T22:14:26,806 DEBUG [RS:0;bd6592c37bb2:40087-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/63842f73863e9e7b9587486b122e6aae/.tmp/C/c3ebf3de6c6240cdb34d8d2cd189aadf as hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/63842f73863e9e7b9587486b122e6aae/C/c3ebf3de6c6240cdb34d8d2cd189aadf 2024-11-24T22:14:26,815 INFO [RS:0;bd6592c37bb2:40087-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 63842f73863e9e7b9587486b122e6aae/C of 63842f73863e9e7b9587486b122e6aae into c3ebf3de6c6240cdb34d8d2cd189aadf(size=11.8 K), total size for store is 11.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-24T22:14:26,816 DEBUG [RS:0;bd6592c37bb2:40087-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 63842f73863e9e7b9587486b122e6aae: 2024-11-24T22:14:26,816 INFO [RS:0;bd6592c37bb2:40087-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732486460633.63842f73863e9e7b9587486b122e6aae., storeName=63842f73863e9e7b9587486b122e6aae/C, priority=13, startTime=1732486466530; duration=0sec 2024-11-24T22:14:26,816 DEBUG [RS:0;bd6592c37bb2:40087-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-24T22:14:26,816 DEBUG [RS:0;bd6592c37bb2:40087-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 63842f73863e9e7b9587486b122e6aae:C 2024-11-24T22:14:26,904 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=40087 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=41 2024-11-24T22:14:26,904 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bd6592c37bb2:0-0 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732486460633.63842f73863e9e7b9587486b122e6aae. 2024-11-24T22:14:26,904 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bd6592c37bb2:0-0 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732486460633.63842f73863e9e7b9587486b122e6aae. as already flushing 2024-11-24T22:14:26,904 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bd6592c37bb2:0-0 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732486460633.63842f73863e9e7b9587486b122e6aae. 2024-11-24T22:14:26,904 ERROR [RS_FLUSH_OPERATIONS-regionserver/bd6592c37bb2:0-0 {event_type=RS_FLUSH_REGIONS, pid=41}] handler.RSProcedureHandler(58): pid=41 java.io.IOException: Unable to complete flush {ENCODED => 63842f73863e9e7b9587486b122e6aae, NAME => 'TestAcidGuarantees,,1732486460633.63842f73863e9e7b9587486b122e6aae.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T22:14:26,904 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bd6592c37bb2:0-0 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=41 java.io.IOException: Unable to complete flush {ENCODED => 63842f73863e9e7b9587486b122e6aae, NAME => 'TestAcidGuarantees,,1732486460633.63842f73863e9e7b9587486b122e6aae.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T22:14:26,905 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37701 {}] master.HMaster(4177): Remote procedure failed, pid=41 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 63842f73863e9e7b9587486b122e6aae, NAME => 'TestAcidGuarantees,,1732486460633.63842f73863e9e7b9587486b122e6aae.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 63842f73863e9e7b9587486b122e6aae, NAME => 'TestAcidGuarantees,,1732486460633.63842f73863e9e7b9587486b122e6aae.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T22:14:26,931 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40087 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=63842f73863e9e7b9587486b122e6aae, server=bd6592c37bb2,40087,1732486423713 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-24T22:14:26,932 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40087 {}] ipc.CallRunner(138): callId: 35 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40206 deadline: 1732486526921, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=63842f73863e9e7b9587486b122e6aae, server=bd6592c37bb2,40087,1732486423713 2024-11-24T22:14:26,932 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732486460633.63842f73863e9e7b9587486b122e6aae., hostname=bd6592c37bb2,40087,1732486423713, seqNum=5 , the old value is region=TestAcidGuarantees,,1732486460633.63842f73863e9e7b9587486b122e6aae., hostname=bd6592c37bb2,40087,1732486423713, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=63842f73863e9e7b9587486b122e6aae, server=bd6592c37bb2,40087,1732486423713 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T22:14:26,932 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732486460633.63842f73863e9e7b9587486b122e6aae., hostname=bd6592c37bb2,40087,1732486423713, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=63842f73863e9e7b9587486b122e6aae, server=bd6592c37bb2,40087,1732486423713 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T22:14:26,932 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732486460633.63842f73863e9e7b9587486b122e6aae., hostname=bd6592c37bb2,40087,1732486423713, seqNum=5 because the exception is null or not the one we care about 2024-11-24T22:14:26,933 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40087 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=63842f73863e9e7b9587486b122e6aae, server=bd6592c37bb2,40087,1732486423713 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-24T22:14:26,933 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40087 {}] ipc.CallRunner(138): callId: 28 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40244 deadline: 1732486526931, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=63842f73863e9e7b9587486b122e6aae, server=bd6592c37bb2,40087,1732486423713 2024-11-24T22:14:26,933 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732486460633.63842f73863e9e7b9587486b122e6aae., hostname=bd6592c37bb2,40087,1732486423713, seqNum=5 , the old value is region=TestAcidGuarantees,,1732486460633.63842f73863e9e7b9587486b122e6aae., hostname=bd6592c37bb2,40087,1732486423713, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=63842f73863e9e7b9587486b122e6aae, server=bd6592c37bb2,40087,1732486423713 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T22:14:26,933 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732486460633.63842f73863e9e7b9587486b122e6aae., hostname=bd6592c37bb2,40087,1732486423713, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=63842f73863e9e7b9587486b122e6aae, server=bd6592c37bb2,40087,1732486423713 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T22:14:26,934 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732486460633.63842f73863e9e7b9587486b122e6aae., hostname=bd6592c37bb2,40087,1732486423713, seqNum=5 because the exception is null or not the one we care about 2024-11-24T22:14:27,015 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40087 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=63842f73863e9e7b9587486b122e6aae, server=bd6592c37bb2,40087,1732486423713 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-24T22:14:27,016 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40087 {}] ipc.CallRunner(138): callId: 28 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40202 deadline: 1732486527011, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=63842f73863e9e7b9587486b122e6aae, server=bd6592c37bb2,40087,1732486423713 2024-11-24T22:14:27,016 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732486460633.63842f73863e9e7b9587486b122e6aae., hostname=bd6592c37bb2,40087,1732486423713, seqNum=5 , the old value is region=TestAcidGuarantees,,1732486460633.63842f73863e9e7b9587486b122e6aae., hostname=bd6592c37bb2,40087,1732486423713, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=63842f73863e9e7b9587486b122e6aae, server=bd6592c37bb2,40087,1732486423713 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T22:14:27,016 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732486460633.63842f73863e9e7b9587486b122e6aae., hostname=bd6592c37bb2,40087,1732486423713, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=63842f73863e9e7b9587486b122e6aae, server=bd6592c37bb2,40087,1732486423713 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T22:14:27,016 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732486460633.63842f73863e9e7b9587486b122e6aae., hostname=bd6592c37bb2,40087,1732486423713, seqNum=5 because the exception is null or not the one we care about 2024-11-24T22:14:27,033 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40087 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=63842f73863e9e7b9587486b122e6aae, server=bd6592c37bb2,40087,1732486423713 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-24T22:14:27,034 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40087 {}] ipc.CallRunner(138): callId: 25 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40226 deadline: 1732486527031, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=63842f73863e9e7b9587486b122e6aae, server=bd6592c37bb2,40087,1732486423713 2024-11-24T22:14:27,034 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40087 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=63842f73863e9e7b9587486b122e6aae, server=bd6592c37bb2,40087,1732486423713 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-24T22:14:27,034 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732486460633.63842f73863e9e7b9587486b122e6aae., hostname=bd6592c37bb2,40087,1732486423713, seqNum=5 , the old value is region=TestAcidGuarantees,,1732486460633.63842f73863e9e7b9587486b122e6aae., hostname=bd6592c37bb2,40087,1732486423713, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=63842f73863e9e7b9587486b122e6aae, server=bd6592c37bb2,40087,1732486423713 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T22:14:27,034 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732486460633.63842f73863e9e7b9587486b122e6aae., hostname=bd6592c37bb2,40087,1732486423713, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=63842f73863e9e7b9587486b122e6aae, server=bd6592c37bb2,40087,1732486423713 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T22:14:27,034 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40087 {}] ipc.CallRunner(138): callId: 24 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40232 deadline: 1732486527031, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=63842f73863e9e7b9587486b122e6aae, server=bd6592c37bb2,40087,1732486423713 2024-11-24T22:14:27,034 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732486460633.63842f73863e9e7b9587486b122e6aae., hostname=bd6592c37bb2,40087,1732486423713, seqNum=5 because the exception is null or not the one we care about 2024-11-24T22:14:27,035 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732486460633.63842f73863e9e7b9587486b122e6aae., hostname=bd6592c37bb2,40087,1732486423713, seqNum=5 , the old value is region=TestAcidGuarantees,,1732486460633.63842f73863e9e7b9587486b122e6aae., hostname=bd6592c37bb2,40087,1732486423713, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=63842f73863e9e7b9587486b122e6aae, server=bd6592c37bb2,40087,1732486423713 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T22:14:27,035 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732486460633.63842f73863e9e7b9587486b122e6aae., hostname=bd6592c37bb2,40087,1732486423713, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=63842f73863e9e7b9587486b122e6aae, server=bd6592c37bb2,40087,1732486423713 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T22:14:27,035 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732486460633.63842f73863e9e7b9587486b122e6aae., hostname=bd6592c37bb2,40087,1732486423713, seqNum=5 because the exception is null or not the one we care about 2024-11-24T22:14:27,061 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=40087 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=41 2024-11-24T22:14:27,061 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bd6592c37bb2:0-1 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732486460633.63842f73863e9e7b9587486b122e6aae. 2024-11-24T22:14:27,061 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bd6592c37bb2:0-1 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732486460633.63842f73863e9e7b9587486b122e6aae. as already flushing 2024-11-24T22:14:27,061 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bd6592c37bb2:0-1 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732486460633.63842f73863e9e7b9587486b122e6aae. 2024-11-24T22:14:27,061 ERROR [RS_FLUSH_OPERATIONS-regionserver/bd6592c37bb2:0-1 {event_type=RS_FLUSH_REGIONS, pid=41}] handler.RSProcedureHandler(58): pid=41 java.io.IOException: Unable to complete flush {ENCODED => 63842f73863e9e7b9587486b122e6aae, NAME => 'TestAcidGuarantees,,1732486460633.63842f73863e9e7b9587486b122e6aae.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T22:14:27,062 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bd6592c37bb2:0-1 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=41 java.io.IOException: Unable to complete flush {ENCODED => 63842f73863e9e7b9587486b122e6aae, NAME => 'TestAcidGuarantees,,1732486460633.63842f73863e9e7b9587486b122e6aae.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T22:14:27,064 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37701 {}] master.HMaster(4177): Remote procedure failed, pid=41 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 63842f73863e9e7b9587486b122e6aae, NAME => 'TestAcidGuarantees,,1732486460633.63842f73863e9e7b9587486b122e6aae.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 63842f73863e9e7b9587486b122e6aae, NAME => 'TestAcidGuarantees,,1732486460633.63842f73863e9e7b9587486b122e6aae.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T22:14:27,101 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37701 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=40 2024-11-24T22:14:27,153 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=85, memsize=38.0 K, hasBloomFilter=true, into tmp file hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/63842f73863e9e7b9587486b122e6aae/.tmp/A/f00867b3407d44fa9ab08e3464c78907 2024-11-24T22:14:27,183 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/63842f73863e9e7b9587486b122e6aae/.tmp/B/e27b972d2a874d369095cb09a68a3f9f is 50, key is test_row_0/B:col10/1732486465856/Put/seqid=0 2024-11-24T22:14:27,218 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=40087 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=41 2024-11-24T22:14:27,218 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45929 is added to blk_1073741957_1133 (size=12001) 2024-11-24T22:14:27,218 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bd6592c37bb2:0-2 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732486460633.63842f73863e9e7b9587486b122e6aae. 2024-11-24T22:14:27,218 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bd6592c37bb2:0-2 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732486460633.63842f73863e9e7b9587486b122e6aae. as already flushing 2024-11-24T22:14:27,218 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bd6592c37bb2:0-2 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732486460633.63842f73863e9e7b9587486b122e6aae. 2024-11-24T22:14:27,218 ERROR [RS_FLUSH_OPERATIONS-regionserver/bd6592c37bb2:0-2 {event_type=RS_FLUSH_REGIONS, pid=41}] handler.RSProcedureHandler(58): pid=41 java.io.IOException: Unable to complete flush {ENCODED => 63842f73863e9e7b9587486b122e6aae, NAME => 'TestAcidGuarantees,,1732486460633.63842f73863e9e7b9587486b122e6aae.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T22:14:27,218 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bd6592c37bb2:0-2 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=41 java.io.IOException: Unable to complete flush {ENCODED => 63842f73863e9e7b9587486b122e6aae, NAME => 'TestAcidGuarantees,,1732486460633.63842f73863e9e7b9587486b122e6aae.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T22:14:27,220 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37701 {}] master.HMaster(4177): Remote procedure failed, pid=41 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 63842f73863e9e7b9587486b122e6aae, NAME => 'TestAcidGuarantees,,1732486460633.63842f73863e9e7b9587486b122e6aae.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 63842f73863e9e7b9587486b122e6aae, NAME => 'TestAcidGuarantees,,1732486460633.63842f73863e9e7b9587486b122e6aae.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T22:14:27,242 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40087 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=63842f73863e9e7b9587486b122e6aae, server=bd6592c37bb2,40087,1732486423713 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-24T22:14:27,245 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40087 {}] ipc.CallRunner(138): callId: 36 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40206 deadline: 1732486527241, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=63842f73863e9e7b9587486b122e6aae, server=bd6592c37bb2,40087,1732486423713 2024-11-24T22:14:27,246 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732486460633.63842f73863e9e7b9587486b122e6aae., hostname=bd6592c37bb2,40087,1732486423713, seqNum=5 , the old value is region=TestAcidGuarantees,,1732486460633.63842f73863e9e7b9587486b122e6aae., hostname=bd6592c37bb2,40087,1732486423713, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=63842f73863e9e7b9587486b122e6aae, server=bd6592c37bb2,40087,1732486423713 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T22:14:27,246 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40087 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=63842f73863e9e7b9587486b122e6aae, server=bd6592c37bb2,40087,1732486423713 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-24T22:14:27,246 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732486460633.63842f73863e9e7b9587486b122e6aae., hostname=bd6592c37bb2,40087,1732486423713, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=63842f73863e9e7b9587486b122e6aae, server=bd6592c37bb2,40087,1732486423713 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T22:14:27,246 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732486460633.63842f73863e9e7b9587486b122e6aae., hostname=bd6592c37bb2,40087,1732486423713, seqNum=5 because the exception is null or not the one we care about 2024-11-24T22:14:27,246 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40087 {}] ipc.CallRunner(138): callId: 29 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40244 deadline: 1732486527241, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=63842f73863e9e7b9587486b122e6aae, server=bd6592c37bb2,40087,1732486423713 2024-11-24T22:14:27,246 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732486460633.63842f73863e9e7b9587486b122e6aae., hostname=bd6592c37bb2,40087,1732486423713, seqNum=5 , the old value is region=TestAcidGuarantees,,1732486460633.63842f73863e9e7b9587486b122e6aae., hostname=bd6592c37bb2,40087,1732486423713, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=63842f73863e9e7b9587486b122e6aae, server=bd6592c37bb2,40087,1732486423713 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T22:14:27,246 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732486460633.63842f73863e9e7b9587486b122e6aae., hostname=bd6592c37bb2,40087,1732486423713, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=63842f73863e9e7b9587486b122e6aae, server=bd6592c37bb2,40087,1732486423713 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T22:14:27,246 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732486460633.63842f73863e9e7b9587486b122e6aae., hostname=bd6592c37bb2,40087,1732486423713, seqNum=5 because the exception is null or not the one we care about 2024-11-24T22:14:27,373 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=40087 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=41 2024-11-24T22:14:27,374 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bd6592c37bb2:0-0 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732486460633.63842f73863e9e7b9587486b122e6aae. 2024-11-24T22:14:27,374 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bd6592c37bb2:0-0 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732486460633.63842f73863e9e7b9587486b122e6aae. as already flushing 2024-11-24T22:14:27,374 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bd6592c37bb2:0-0 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732486460633.63842f73863e9e7b9587486b122e6aae. 2024-11-24T22:14:27,374 ERROR [RS_FLUSH_OPERATIONS-regionserver/bd6592c37bb2:0-0 {event_type=RS_FLUSH_REGIONS, pid=41}] handler.RSProcedureHandler(58): pid=41 java.io.IOException: Unable to complete flush {ENCODED => 63842f73863e9e7b9587486b122e6aae, NAME => 'TestAcidGuarantees,,1732486460633.63842f73863e9e7b9587486b122e6aae.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T22:14:27,374 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bd6592c37bb2:0-0 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=41 java.io.IOException: Unable to complete flush {ENCODED => 63842f73863e9e7b9587486b122e6aae, NAME => 'TestAcidGuarantees,,1732486460633.63842f73863e9e7b9587486b122e6aae.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T22:14:27,375 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37701 {}] master.HMaster(4177): Remote procedure failed, pid=41 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 63842f73863e9e7b9587486b122e6aae, NAME => 'TestAcidGuarantees,,1732486460633.63842f73863e9e7b9587486b122e6aae.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 63842f73863e9e7b9587486b122e6aae, NAME => 'TestAcidGuarantees,,1732486460633.63842f73863e9e7b9587486b122e6aae.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T22:14:27,528 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=40087 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=41 2024-11-24T22:14:27,529 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bd6592c37bb2:0-1 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732486460633.63842f73863e9e7b9587486b122e6aae. 2024-11-24T22:14:27,529 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bd6592c37bb2:0-1 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732486460633.63842f73863e9e7b9587486b122e6aae. as already flushing 2024-11-24T22:14:27,529 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bd6592c37bb2:0-1 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732486460633.63842f73863e9e7b9587486b122e6aae. 2024-11-24T22:14:27,529 ERROR [RS_FLUSH_OPERATIONS-regionserver/bd6592c37bb2:0-1 {event_type=RS_FLUSH_REGIONS, pid=41}] handler.RSProcedureHandler(58): pid=41 java.io.IOException: Unable to complete flush {ENCODED => 63842f73863e9e7b9587486b122e6aae, NAME => 'TestAcidGuarantees,,1732486460633.63842f73863e9e7b9587486b122e6aae.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T22:14:27,529 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bd6592c37bb2:0-1 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=41 java.io.IOException: Unable to complete flush {ENCODED => 63842f73863e9e7b9587486b122e6aae, NAME => 'TestAcidGuarantees,,1732486460633.63842f73863e9e7b9587486b122e6aae.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T22:14:27,530 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37701 {}] master.HMaster(4177): Remote procedure failed, pid=41 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 63842f73863e9e7b9587486b122e6aae, NAME => 'TestAcidGuarantees,,1732486460633.63842f73863e9e7b9587486b122e6aae.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 63842f73863e9e7b9587486b122e6aae, NAME => 'TestAcidGuarantees,,1732486460633.63842f73863e9e7b9587486b122e6aae.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T22:14:27,619 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=38.02 KB at sequenceid=85 (bloomFilter=true), to=hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/63842f73863e9e7b9587486b122e6aae/.tmp/B/e27b972d2a874d369095cb09a68a3f9f 2024-11-24T22:14:27,645 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/63842f73863e9e7b9587486b122e6aae/.tmp/C/6897f56dbe744713beb5afa1e154ea02 is 50, key is test_row_0/C:col10/1732486465856/Put/seqid=0 2024-11-24T22:14:27,676 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45929 is added to blk_1073741958_1134 (size=12001) 2024-11-24T22:14:27,683 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=38.02 KB at sequenceid=85 (bloomFilter=true), to=hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/63842f73863e9e7b9587486b122e6aae/.tmp/C/6897f56dbe744713beb5afa1e154ea02 2024-11-24T22:14:27,683 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=40087 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=41 2024-11-24T22:14:27,684 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bd6592c37bb2:0-2 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732486460633.63842f73863e9e7b9587486b122e6aae. 2024-11-24T22:14:27,684 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bd6592c37bb2:0-2 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732486460633.63842f73863e9e7b9587486b122e6aae. as already flushing 2024-11-24T22:14:27,684 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bd6592c37bb2:0-2 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732486460633.63842f73863e9e7b9587486b122e6aae. 2024-11-24T22:14:27,684 ERROR [RS_FLUSH_OPERATIONS-regionserver/bd6592c37bb2:0-2 {event_type=RS_FLUSH_REGIONS, pid=41}] handler.RSProcedureHandler(58): pid=41 java.io.IOException: Unable to complete flush {ENCODED => 63842f73863e9e7b9587486b122e6aae, NAME => 'TestAcidGuarantees,,1732486460633.63842f73863e9e7b9587486b122e6aae.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T22:14:27,685 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bd6592c37bb2:0-2 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=41 java.io.IOException: Unable to complete flush {ENCODED => 63842f73863e9e7b9587486b122e6aae, NAME => 'TestAcidGuarantees,,1732486460633.63842f73863e9e7b9587486b122e6aae.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T22:14:27,685 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37701 {}] master.HMaster(4177): Remote procedure failed, pid=41 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 63842f73863e9e7b9587486b122e6aae, NAME => 'TestAcidGuarantees,,1732486460633.63842f73863e9e7b9587486b122e6aae.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 63842f73863e9e7b9587486b122e6aae, NAME => 'TestAcidGuarantees,,1732486460633.63842f73863e9e7b9587486b122e6aae.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T22:14:27,691 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/63842f73863e9e7b9587486b122e6aae/.tmp/A/f00867b3407d44fa9ab08e3464c78907 as hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/63842f73863e9e7b9587486b122e6aae/A/f00867b3407d44fa9ab08e3464c78907 2024-11-24T22:14:27,698 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/63842f73863e9e7b9587486b122e6aae/A/f00867b3407d44fa9ab08e3464c78907, entries=150, sequenceid=85, filesize=30.2 K 2024-11-24T22:14:27,700 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/63842f73863e9e7b9587486b122e6aae/.tmp/B/e27b972d2a874d369095cb09a68a3f9f as hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/63842f73863e9e7b9587486b122e6aae/B/e27b972d2a874d369095cb09a68a3f9f 2024-11-24T22:14:27,708 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/63842f73863e9e7b9587486b122e6aae/B/e27b972d2a874d369095cb09a68a3f9f, entries=150, sequenceid=85, filesize=11.7 K 2024-11-24T22:14:27,710 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/63842f73863e9e7b9587486b122e6aae/.tmp/C/6897f56dbe744713beb5afa1e154ea02 as hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/63842f73863e9e7b9587486b122e6aae/C/6897f56dbe744713beb5afa1e154ea02 2024-11-24T22:14:27,720 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/63842f73863e9e7b9587486b122e6aae/C/6897f56dbe744713beb5afa1e154ea02, entries=150, sequenceid=85, filesize=11.7 K 2024-11-24T22:14:27,721 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~114.05 KB/116790, heapSize ~339.38 KB/347520, currentSize=60.38 KB/61830 for 63842f73863e9e7b9587486b122e6aae in 1176ms, sequenceid=85, compaction requested=false 2024-11-24T22:14:27,721 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 63842f73863e9e7b9587486b122e6aae: 2024-11-24T22:14:27,763 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40087 {}] regionserver.HRegion(8855): Flush requested on 63842f73863e9e7b9587486b122e6aae 2024-11-24T22:14:27,763 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 63842f73863e9e7b9587486b122e6aae 3/3 column families, dataSize=67.09 KB heapSize=199.97 KB 2024-11-24T22:14:27,770 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 63842f73863e9e7b9587486b122e6aae, store=A 2024-11-24T22:14:27,770 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-24T22:14:27,770 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 63842f73863e9e7b9587486b122e6aae, store=B 2024-11-24T22:14:27,770 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-24T22:14:27,770 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 63842f73863e9e7b9587486b122e6aae, store=C 2024-11-24T22:14:27,770 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-24T22:14:27,795 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411241184d2c472984a59a5ea93a8a3fb1025_63842f73863e9e7b9587486b122e6aae is 50, key is test_row_0/A:col10/1732486466601/Put/seqid=0 2024-11-24T22:14:27,825 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45929 is added to blk_1073741959_1135 (size=14594) 2024-11-24T22:14:27,826 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T22:14:27,832 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411241184d2c472984a59a5ea93a8a3fb1025_63842f73863e9e7b9587486b122e6aae to hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411241184d2c472984a59a5ea93a8a3fb1025_63842f73863e9e7b9587486b122e6aae 2024-11-24T22:14:27,834 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/63842f73863e9e7b9587486b122e6aae/.tmp/A/fa7b82d0c1014b23862f5a4f118a9a59, store: [table=TestAcidGuarantees family=A region=63842f73863e9e7b9587486b122e6aae] 2024-11-24T22:14:27,834 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/63842f73863e9e7b9587486b122e6aae/.tmp/A/fa7b82d0c1014b23862f5a4f118a9a59 is 175, key is test_row_0/A:col10/1732486466601/Put/seqid=0 2024-11-24T22:14:27,838 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=40087 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=41 2024-11-24T22:14:27,839 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bd6592c37bb2:0-0 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732486460633.63842f73863e9e7b9587486b122e6aae. 2024-11-24T22:14:27,839 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bd6592c37bb2:0-0 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732486460633.63842f73863e9e7b9587486b122e6aae. as already flushing 2024-11-24T22:14:27,839 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bd6592c37bb2:0-0 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732486460633.63842f73863e9e7b9587486b122e6aae. 2024-11-24T22:14:27,839 ERROR [RS_FLUSH_OPERATIONS-regionserver/bd6592c37bb2:0-0 {event_type=RS_FLUSH_REGIONS, pid=41}] handler.RSProcedureHandler(58): pid=41 java.io.IOException: Unable to complete flush {ENCODED => 63842f73863e9e7b9587486b122e6aae, NAME => 'TestAcidGuarantees,,1732486460633.63842f73863e9e7b9587486b122e6aae.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T22:14:27,839 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bd6592c37bb2:0-0 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=41 java.io.IOException: Unable to complete flush {ENCODED => 63842f73863e9e7b9587486b122e6aae, NAME => 'TestAcidGuarantees,,1732486460633.63842f73863e9e7b9587486b122e6aae.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T22:14:27,840 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37701 {}] master.HMaster(4177): Remote procedure failed, pid=41 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 63842f73863e9e7b9587486b122e6aae, NAME => 'TestAcidGuarantees,,1732486460633.63842f73863e9e7b9587486b122e6aae.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 63842f73863e9e7b9587486b122e6aae, NAME => 'TestAcidGuarantees,,1732486460633.63842f73863e9e7b9587486b122e6aae.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T22:14:27,845 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40087 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=63842f73863e9e7b9587486b122e6aae, server=bd6592c37bb2,40087,1732486423713 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-24T22:14:27,845 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40087 {}] ipc.CallRunner(138): callId: 38 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40244 deadline: 1732486527838, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=63842f73863e9e7b9587486b122e6aae, server=bd6592c37bb2,40087,1732486423713 2024-11-24T22:14:27,846 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40087 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=63842f73863e9e7b9587486b122e6aae, server=bd6592c37bb2,40087,1732486423713 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-24T22:14:27,846 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732486460633.63842f73863e9e7b9587486b122e6aae., hostname=bd6592c37bb2,40087,1732486423713, seqNum=5 , the old value is region=TestAcidGuarantees,,1732486460633.63842f73863e9e7b9587486b122e6aae., hostname=bd6592c37bb2,40087,1732486423713, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=63842f73863e9e7b9587486b122e6aae, server=bd6592c37bb2,40087,1732486423713 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T22:14:27,846 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40087 {}] ipc.CallRunner(138): callId: 46 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40206 deadline: 1732486527845, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=63842f73863e9e7b9587486b122e6aae, server=bd6592c37bb2,40087,1732486423713 2024-11-24T22:14:27,846 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732486460633.63842f73863e9e7b9587486b122e6aae., hostname=bd6592c37bb2,40087,1732486423713, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=63842f73863e9e7b9587486b122e6aae, server=bd6592c37bb2,40087,1732486423713 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T22:14:27,846 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732486460633.63842f73863e9e7b9587486b122e6aae., hostname=bd6592c37bb2,40087,1732486423713, seqNum=5 because the exception is null or not the one we care about 2024-11-24T22:14:27,846 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732486460633.63842f73863e9e7b9587486b122e6aae., hostname=bd6592c37bb2,40087,1732486423713, seqNum=5 , the old value is region=TestAcidGuarantees,,1732486460633.63842f73863e9e7b9587486b122e6aae., hostname=bd6592c37bb2,40087,1732486423713, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=63842f73863e9e7b9587486b122e6aae, server=bd6592c37bb2,40087,1732486423713 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T22:14:27,846 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732486460633.63842f73863e9e7b9587486b122e6aae., hostname=bd6592c37bb2,40087,1732486423713, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=63842f73863e9e7b9587486b122e6aae, server=bd6592c37bb2,40087,1732486423713 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T22:14:27,846 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732486460633.63842f73863e9e7b9587486b122e6aae., hostname=bd6592c37bb2,40087,1732486423713, seqNum=5 because the exception is null or not the one we care about 2024-11-24T22:14:27,871 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45929 is added to blk_1073741960_1136 (size=39549) 2024-11-24T22:14:27,874 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=102, memsize=24.6 K, hasBloomFilter=true, into tmp file hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/63842f73863e9e7b9587486b122e6aae/.tmp/A/fa7b82d0c1014b23862f5a4f118a9a59 2024-11-24T22:14:27,893 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/63842f73863e9e7b9587486b122e6aae/.tmp/B/df8f192c6e7149afb3fabf9e6698e187 is 50, key is test_row_0/B:col10/1732486466601/Put/seqid=0 2024-11-24T22:14:27,942 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45929 is added to blk_1073741961_1137 (size=12001) 2024-11-24T22:14:27,943 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=24.60 KB at sequenceid=102 (bloomFilter=true), to=hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/63842f73863e9e7b9587486b122e6aae/.tmp/B/df8f192c6e7149afb3fabf9e6698e187 2024-11-24T22:14:27,954 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40087 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=63842f73863e9e7b9587486b122e6aae, server=bd6592c37bb2,40087,1732486423713 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-24T22:14:27,954 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40087 {}] ipc.CallRunner(138): callId: 39 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40244 deadline: 1732486527951, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=63842f73863e9e7b9587486b122e6aae, server=bd6592c37bb2,40087,1732486423713 2024-11-24T22:14:27,954 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40087 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=63842f73863e9e7b9587486b122e6aae, server=bd6592c37bb2,40087,1732486423713 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-24T22:14:27,955 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732486460633.63842f73863e9e7b9587486b122e6aae., hostname=bd6592c37bb2,40087,1732486423713, seqNum=5 , the old value is region=TestAcidGuarantees,,1732486460633.63842f73863e9e7b9587486b122e6aae., hostname=bd6592c37bb2,40087,1732486423713, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=63842f73863e9e7b9587486b122e6aae, server=bd6592c37bb2,40087,1732486423713 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T22:14:27,955 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40087 {}] ipc.CallRunner(138): callId: 47 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40206 deadline: 1732486527951, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=63842f73863e9e7b9587486b122e6aae, server=bd6592c37bb2,40087,1732486423713 2024-11-24T22:14:27,955 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732486460633.63842f73863e9e7b9587486b122e6aae., hostname=bd6592c37bb2,40087,1732486423713, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=63842f73863e9e7b9587486b122e6aae, server=bd6592c37bb2,40087,1732486423713 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T22:14:27,955 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732486460633.63842f73863e9e7b9587486b122e6aae., hostname=bd6592c37bb2,40087,1732486423713, seqNum=5 because the exception is null or not the one we care about 2024-11-24T22:14:27,955 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732486460633.63842f73863e9e7b9587486b122e6aae., hostname=bd6592c37bb2,40087,1732486423713, seqNum=5 , the old value is region=TestAcidGuarantees,,1732486460633.63842f73863e9e7b9587486b122e6aae., hostname=bd6592c37bb2,40087,1732486423713, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=63842f73863e9e7b9587486b122e6aae, server=bd6592c37bb2,40087,1732486423713 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T22:14:27,955 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732486460633.63842f73863e9e7b9587486b122e6aae., hostname=bd6592c37bb2,40087,1732486423713, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=63842f73863e9e7b9587486b122e6aae, server=bd6592c37bb2,40087,1732486423713 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T22:14:27,955 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732486460633.63842f73863e9e7b9587486b122e6aae., hostname=bd6592c37bb2,40087,1732486423713, seqNum=5 because the exception is null or not the one we care about 2024-11-24T22:14:27,971 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/63842f73863e9e7b9587486b122e6aae/.tmp/C/17481ead002944a3b87ffd04ccafc446 is 50, key is test_row_0/C:col10/1732486466601/Put/seqid=0 2024-11-24T22:14:27,992 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=40087 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=41 2024-11-24T22:14:27,992 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bd6592c37bb2:0-1 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732486460633.63842f73863e9e7b9587486b122e6aae. 2024-11-24T22:14:27,992 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bd6592c37bb2:0-1 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732486460633.63842f73863e9e7b9587486b122e6aae. as already flushing 2024-11-24T22:14:27,992 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bd6592c37bb2:0-1 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732486460633.63842f73863e9e7b9587486b122e6aae. 2024-11-24T22:14:27,992 ERROR [RS_FLUSH_OPERATIONS-regionserver/bd6592c37bb2:0-1 {event_type=RS_FLUSH_REGIONS, pid=41}] handler.RSProcedureHandler(58): pid=41 java.io.IOException: Unable to complete flush {ENCODED => 63842f73863e9e7b9587486b122e6aae, NAME => 'TestAcidGuarantees,,1732486460633.63842f73863e9e7b9587486b122e6aae.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T22:14:27,993 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bd6592c37bb2:0-1 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=41 java.io.IOException: Unable to complete flush {ENCODED => 63842f73863e9e7b9587486b122e6aae, NAME => 'TestAcidGuarantees,,1732486460633.63842f73863e9e7b9587486b122e6aae.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T22:14:27,994 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37701 {}] master.HMaster(4177): Remote procedure failed, pid=41 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 63842f73863e9e7b9587486b122e6aae, NAME => 'TestAcidGuarantees,,1732486460633.63842f73863e9e7b9587486b122e6aae.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 63842f73863e9e7b9587486b122e6aae, NAME => 'TestAcidGuarantees,,1732486460633.63842f73863e9e7b9587486b122e6aae.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T22:14:28,003 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45929 is added to blk_1073741962_1138 (size=12001) 2024-11-24T22:14:28,004 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=24.60 KB at sequenceid=102 (bloomFilter=true), to=hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/63842f73863e9e7b9587486b122e6aae/.tmp/C/17481ead002944a3b87ffd04ccafc446 2024-11-24T22:14:28,013 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/63842f73863e9e7b9587486b122e6aae/.tmp/A/fa7b82d0c1014b23862f5a4f118a9a59 as hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/63842f73863e9e7b9587486b122e6aae/A/fa7b82d0c1014b23862f5a4f118a9a59 2024-11-24T22:14:28,019 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/63842f73863e9e7b9587486b122e6aae/A/fa7b82d0c1014b23862f5a4f118a9a59, entries=200, sequenceid=102, filesize=38.6 K 2024-11-24T22:14:28,022 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/63842f73863e9e7b9587486b122e6aae/.tmp/B/df8f192c6e7149afb3fabf9e6698e187 as hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/63842f73863e9e7b9587486b122e6aae/B/df8f192c6e7149afb3fabf9e6698e187 2024-11-24T22:14:28,022 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40087 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=63842f73863e9e7b9587486b122e6aae, server=bd6592c37bb2,40087,1732486423713 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-24T22:14:28,028 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/63842f73863e9e7b9587486b122e6aae/B/df8f192c6e7149afb3fabf9e6698e187, entries=150, sequenceid=102, filesize=11.7 K 2024-11-24T22:14:28,028 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40087 {}] ipc.CallRunner(138): callId: 29 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40202 deadline: 1732486528021, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=63842f73863e9e7b9587486b122e6aae, server=bd6592c37bb2,40087,1732486423713 2024-11-24T22:14:28,029 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/63842f73863e9e7b9587486b122e6aae/.tmp/C/17481ead002944a3b87ffd04ccafc446 as hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/63842f73863e9e7b9587486b122e6aae/C/17481ead002944a3b87ffd04ccafc446 2024-11-24T22:14:28,029 WARN [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRpcRetryingCaller(177): Call to bd6592c37bb2,40087,1732486423713 for 'test_row_0' in 63842f73863e9e7b9587486b122e6aae of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2168 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=63842f73863e9e7b9587486b122e6aae, server=bd6592c37bb2,40087,1732486423713 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor44.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=63842f73863e9e7b9587486b122e6aae, server=bd6592c37bb2,40087,1732486423713 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-24T22:14:28,030 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732486460633.63842f73863e9e7b9587486b122e6aae., hostname=bd6592c37bb2,40087,1732486423713, seqNum=5 , the old value is region=TestAcidGuarantees,,1732486460633.63842f73863e9e7b9587486b122e6aae., hostname=bd6592c37bb2,40087,1732486423713, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=63842f73863e9e7b9587486b122e6aae, server=bd6592c37bb2,40087,1732486423713 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T22:14:28,030 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732486460633.63842f73863e9e7b9587486b122e6aae., hostname=bd6592c37bb2,40087,1732486423713, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=63842f73863e9e7b9587486b122e6aae, server=bd6592c37bb2,40087,1732486423713 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T22:14:28,030 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732486460633.63842f73863e9e7b9587486b122e6aae., hostname=bd6592c37bb2,40087,1732486423713, seqNum=5 because the exception is null or not the one we care about 2024-11-24T22:14:28,034 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/63842f73863e9e7b9587486b122e6aae/C/17481ead002944a3b87ffd04ccafc446, entries=150, sequenceid=102, filesize=11.7 K 2024-11-24T22:14:28,035 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~73.80 KB/75570, heapSize ~219.84 KB/225120, currentSize=100.63 KB/103050 for 63842f73863e9e7b9587486b122e6aae in 272ms, sequenceid=102, compaction requested=true 2024-11-24T22:14:28,036 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 63842f73863e9e7b9587486b122e6aae: 2024-11-24T22:14:28,036 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 63842f73863e9e7b9587486b122e6aae:A, priority=-2147483648, current under compaction store size is 1 2024-11-24T22:14:28,036 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-24T22:14:28,036 DEBUG [RS:0;bd6592c37bb2:40087-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-24T22:14:28,036 DEBUG [RS:0;bd6592c37bb2:40087-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-24T22:14:28,036 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 63842f73863e9e7b9587486b122e6aae:B, priority=-2147483648, current under compaction store size is 2 2024-11-24T22:14:28,036 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-24T22:14:28,036 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 63842f73863e9e7b9587486b122e6aae:C, priority=-2147483648, current under compaction store size is 3 2024-11-24T22:14:28,036 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-24T22:14:28,037 DEBUG [RS:0;bd6592c37bb2:40087-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36106 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-24T22:14:28,038 DEBUG [RS:0;bd6592c37bb2:40087-longCompactions-0 {}] regionserver.HStore(1541): 63842f73863e9e7b9587486b122e6aae/B is initiating minor compaction (all files) 2024-11-24T22:14:28,038 INFO [RS:0;bd6592c37bb2:40087-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 63842f73863e9e7b9587486b122e6aae/B in TestAcidGuarantees,,1732486460633.63842f73863e9e7b9587486b122e6aae. 2024-11-24T22:14:28,038 INFO [RS:0;bd6592c37bb2:40087-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/63842f73863e9e7b9587486b122e6aae/B/b4285931220446c38584c66287d3a5b9, hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/63842f73863e9e7b9587486b122e6aae/B/e27b972d2a874d369095cb09a68a3f9f, hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/63842f73863e9e7b9587486b122e6aae/B/df8f192c6e7149afb3fabf9e6698e187] into tmpdir=hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/63842f73863e9e7b9587486b122e6aae/.tmp, totalSize=35.3 K 2024-11-24T22:14:28,039 DEBUG [RS:0;bd6592c37bb2:40087-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 101562 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-24T22:14:28,039 DEBUG [RS:0;bd6592c37bb2:40087-shortCompactions-0 {}] regionserver.HStore(1541): 63842f73863e9e7b9587486b122e6aae/A is initiating minor compaction (all files) 2024-11-24T22:14:28,039 INFO [RS:0;bd6592c37bb2:40087-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 63842f73863e9e7b9587486b122e6aae/A in TestAcidGuarantees,,1732486460633.63842f73863e9e7b9587486b122e6aae. 2024-11-24T22:14:28,039 DEBUG [RS:0;bd6592c37bb2:40087-longCompactions-0 {}] compactions.Compactor(225): Compacting b4285931220446c38584c66287d3a5b9, keycount=150, bloomtype=ROW, size=11.8 K, encoding=NONE, compression=NONE, seqNum=65, earliestPutTs=1732486465173 2024-11-24T22:14:28,039 INFO [RS:0;bd6592c37bb2:40087-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/63842f73863e9e7b9587486b122e6aae/A/61c5cebf766f4aa09183bcfd524ae066, hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/63842f73863e9e7b9587486b122e6aae/A/f00867b3407d44fa9ab08e3464c78907, hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/63842f73863e9e7b9587486b122e6aae/A/fa7b82d0c1014b23862f5a4f118a9a59] into tmpdir=hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/63842f73863e9e7b9587486b122e6aae/.tmp, totalSize=99.2 K 2024-11-24T22:14:28,039 INFO [RS:0;bd6592c37bb2:40087-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=13 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1732486460633.63842f73863e9e7b9587486b122e6aae. 2024-11-24T22:14:28,039 DEBUG [RS:0;bd6592c37bb2:40087-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1732486460633.63842f73863e9e7b9587486b122e6aae. files: [hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/63842f73863e9e7b9587486b122e6aae/A/61c5cebf766f4aa09183bcfd524ae066, hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/63842f73863e9e7b9587486b122e6aae/A/f00867b3407d44fa9ab08e3464c78907, hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/63842f73863e9e7b9587486b122e6aae/A/fa7b82d0c1014b23862f5a4f118a9a59] 2024-11-24T22:14:28,042 DEBUG [RS:0;bd6592c37bb2:40087-longCompactions-0 {}] compactions.Compactor(225): Compacting e27b972d2a874d369095cb09a68a3f9f, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=85, earliestPutTs=1732486465856 2024-11-24T22:14:28,042 DEBUG [RS:0;bd6592c37bb2:40087-shortCompactions-0 {}] compactions.Compactor(225): Compacting 61c5cebf766f4aa09183bcfd524ae066, keycount=150, bloomtype=ROW, size=30.3 K, encoding=NONE, compression=NONE, seqNum=65, earliestPutTs=1732486465173 2024-11-24T22:14:28,042 DEBUG [RS:0;bd6592c37bb2:40087-longCompactions-0 {}] compactions.Compactor(225): Compacting df8f192c6e7149afb3fabf9e6698e187, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=102, earliestPutTs=1732486466601 2024-11-24T22:14:28,042 DEBUG [RS:0;bd6592c37bb2:40087-shortCompactions-0 {}] compactions.Compactor(225): Compacting f00867b3407d44fa9ab08e3464c78907, keycount=150, bloomtype=ROW, size=30.2 K, encoding=NONE, compression=NONE, seqNum=85, earliestPutTs=1732486465856 2024-11-24T22:14:28,043 DEBUG [RS:0;bd6592c37bb2:40087-shortCompactions-0 {}] compactions.Compactor(225): Compacting fa7b82d0c1014b23862f5a4f118a9a59, keycount=200, bloomtype=ROW, size=38.6 K, encoding=NONE, compression=NONE, seqNum=102, earliestPutTs=1732486466592 2024-11-24T22:14:28,045 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40087 {}] regionserver.HRegion(8855): Flush requested on 63842f73863e9e7b9587486b122e6aae 2024-11-24T22:14:28,047 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 63842f73863e9e7b9587486b122e6aae 3/3 column families, dataSize=114.05 KB heapSize=339.42 KB 2024-11-24T22:14:28,048 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 63842f73863e9e7b9587486b122e6aae, store=A 2024-11-24T22:14:28,052 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-24T22:14:28,052 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 63842f73863e9e7b9587486b122e6aae, store=B 2024-11-24T22:14:28,052 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-24T22:14:28,052 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 63842f73863e9e7b9587486b122e6aae, store=C 2024-11-24T22:14:28,052 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-24T22:14:28,064 INFO [RS:0;bd6592c37bb2:40087-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 63842f73863e9e7b9587486b122e6aae#B#compaction#117 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-24T22:14:28,065 DEBUG [RS:0;bd6592c37bb2:40087-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/63842f73863e9e7b9587486b122e6aae/.tmp/B/85d047d5e1164e14a0ecf07a70e8146c is 50, key is test_row_0/B:col10/1732486466601/Put/seqid=0 2024-11-24T22:14:28,077 INFO [RS:0;bd6592c37bb2:40087-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=63842f73863e9e7b9587486b122e6aae] 2024-11-24T22:14:28,095 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40087 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=63842f73863e9e7b9587486b122e6aae, server=bd6592c37bb2,40087,1732486423713 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-24T22:14:28,095 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40087 {}] ipc.CallRunner(138): callId: 31 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40226 deadline: 1732486528088, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=63842f73863e9e7b9587486b122e6aae, server=bd6592c37bb2,40087,1732486423713 2024-11-24T22:14:28,096 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732486460633.63842f73863e9e7b9587486b122e6aae., hostname=bd6592c37bb2,40087,1732486423713, seqNum=5 , the old value is region=TestAcidGuarantees,,1732486460633.63842f73863e9e7b9587486b122e6aae., hostname=bd6592c37bb2,40087,1732486423713, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=63842f73863e9e7b9587486b122e6aae, server=bd6592c37bb2,40087,1732486423713 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T22:14:28,096 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732486460633.63842f73863e9e7b9587486b122e6aae., hostname=bd6592c37bb2,40087,1732486423713, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=63842f73863e9e7b9587486b122e6aae, server=bd6592c37bb2,40087,1732486423713 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T22:14:28,096 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732486460633.63842f73863e9e7b9587486b122e6aae., hostname=bd6592c37bb2,40087,1732486423713, seqNum=5 because the exception is null or not the one we care about 2024-11-24T22:14:28,099 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40087 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=63842f73863e9e7b9587486b122e6aae, server=bd6592c37bb2,40087,1732486423713 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-24T22:14:28,099 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40087 {}] ipc.CallRunner(138): callId: 31 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40232 deadline: 1732486528095, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=63842f73863e9e7b9587486b122e6aae, server=bd6592c37bb2,40087,1732486423713 2024-11-24T22:14:28,099 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732486460633.63842f73863e9e7b9587486b122e6aae., hostname=bd6592c37bb2,40087,1732486423713, seqNum=5 , the old value is region=TestAcidGuarantees,,1732486460633.63842f73863e9e7b9587486b122e6aae., hostname=bd6592c37bb2,40087,1732486423713, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=63842f73863e9e7b9587486b122e6aae, server=bd6592c37bb2,40087,1732486423713 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T22:14:28,099 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732486460633.63842f73863e9e7b9587486b122e6aae., hostname=bd6592c37bb2,40087,1732486423713, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=63842f73863e9e7b9587486b122e6aae, server=bd6592c37bb2,40087,1732486423713 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T22:14:28,100 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732486460633.63842f73863e9e7b9587486b122e6aae., hostname=bd6592c37bb2,40087,1732486423713, seqNum=5 because the exception is null or not the one we care about 2024-11-24T22:14:28,111 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37701 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=40 2024-11-24T22:14:28,113 DEBUG [RS:0;bd6592c37bb2:40087-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e20241124dbd8300f89884704b808567035da2709_63842f73863e9e7b9587486b122e6aae store=[table=TestAcidGuarantees family=A region=63842f73863e9e7b9587486b122e6aae] 2024-11-24T22:14:28,116 DEBUG [RS:0;bd6592c37bb2:40087-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e20241124dbd8300f89884704b808567035da2709_63842f73863e9e7b9587486b122e6aae, store=[table=TestAcidGuarantees family=A region=63842f73863e9e7b9587486b122e6aae] 2024-11-24T22:14:28,117 DEBUG [RS:0;bd6592c37bb2:40087-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241124dbd8300f89884704b808567035da2709_63842f73863e9e7b9587486b122e6aae because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=63842f73863e9e7b9587486b122e6aae] 2024-11-24T22:14:28,125 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024112426339100592e4032b4171064d82de25b_63842f73863e9e7b9587486b122e6aae is 50, key is test_row_0/A:col10/1732486467804/Put/seqid=0 2024-11-24T22:14:28,149 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=40087 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=41 2024-11-24T22:14:28,150 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bd6592c37bb2:0-2 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732486460633.63842f73863e9e7b9587486b122e6aae. 2024-11-24T22:14:28,150 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bd6592c37bb2:0-2 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732486460633.63842f73863e9e7b9587486b122e6aae. as already flushing 2024-11-24T22:14:28,150 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bd6592c37bb2:0-2 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732486460633.63842f73863e9e7b9587486b122e6aae. 2024-11-24T22:14:28,150 ERROR [RS_FLUSH_OPERATIONS-regionserver/bd6592c37bb2:0-2 {event_type=RS_FLUSH_REGIONS, pid=41}] handler.RSProcedureHandler(58): pid=41 java.io.IOException: Unable to complete flush {ENCODED => 63842f73863e9e7b9587486b122e6aae, NAME => 'TestAcidGuarantees,,1732486460633.63842f73863e9e7b9587486b122e6aae.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T22:14:28,150 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bd6592c37bb2:0-2 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=41 java.io.IOException: Unable to complete flush {ENCODED => 63842f73863e9e7b9587486b122e6aae, NAME => 'TestAcidGuarantees,,1732486460633.63842f73863e9e7b9587486b122e6aae.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T22:14:28,151 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37701 {}] master.HMaster(4177): Remote procedure failed, pid=41 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 63842f73863e9e7b9587486b122e6aae, NAME => 'TestAcidGuarantees,,1732486460633.63842f73863e9e7b9587486b122e6aae.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 63842f73863e9e7b9587486b122e6aae, NAME => 'TestAcidGuarantees,,1732486460633.63842f73863e9e7b9587486b122e6aae.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T22:14:28,152 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45929 is added to blk_1073741963_1139 (size=12207) 2024-11-24T22:14:28,166 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40087 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=63842f73863e9e7b9587486b122e6aae, server=bd6592c37bb2,40087,1732486423713 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-24T22:14:28,166 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40087 {}] ipc.CallRunner(138): callId: 40 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40244 deadline: 1732486528161, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=63842f73863e9e7b9587486b122e6aae, server=bd6592c37bb2,40087,1732486423713 2024-11-24T22:14:28,166 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732486460633.63842f73863e9e7b9587486b122e6aae., hostname=bd6592c37bb2,40087,1732486423713, seqNum=5 , the old value is region=TestAcidGuarantees,,1732486460633.63842f73863e9e7b9587486b122e6aae., hostname=bd6592c37bb2,40087,1732486423713, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=63842f73863e9e7b9587486b122e6aae, server=bd6592c37bb2,40087,1732486423713 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T22:14:28,166 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732486460633.63842f73863e9e7b9587486b122e6aae., hostname=bd6592c37bb2,40087,1732486423713, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=63842f73863e9e7b9587486b122e6aae, server=bd6592c37bb2,40087,1732486423713 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T22:14:28,166 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732486460633.63842f73863e9e7b9587486b122e6aae., hostname=bd6592c37bb2,40087,1732486423713, seqNum=5 because the exception is null or not the one we care about 2024-11-24T22:14:28,167 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40087 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=63842f73863e9e7b9587486b122e6aae, server=bd6592c37bb2,40087,1732486423713 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-24T22:14:28,167 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40087 {}] ipc.CallRunner(138): callId: 48 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40206 deadline: 1732486528161, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=63842f73863e9e7b9587486b122e6aae, server=bd6592c37bb2,40087,1732486423713 2024-11-24T22:14:28,167 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732486460633.63842f73863e9e7b9587486b122e6aae., hostname=bd6592c37bb2,40087,1732486423713, seqNum=5 , the old value is region=TestAcidGuarantees,,1732486460633.63842f73863e9e7b9587486b122e6aae., hostname=bd6592c37bb2,40087,1732486423713, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=63842f73863e9e7b9587486b122e6aae, server=bd6592c37bb2,40087,1732486423713 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T22:14:28,167 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732486460633.63842f73863e9e7b9587486b122e6aae., hostname=bd6592c37bb2,40087,1732486423713, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=63842f73863e9e7b9587486b122e6aae, server=bd6592c37bb2,40087,1732486423713 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T22:14:28,167 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732486460633.63842f73863e9e7b9587486b122e6aae., hostname=bd6592c37bb2,40087,1732486423713, seqNum=5 because the exception is null or not the one we care about 2024-11-24T22:14:28,199 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45929 is added to blk_1073741965_1141 (size=12154) 2024-11-24T22:14:28,199 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45929 is added to blk_1073741964_1140 (size=4469) 2024-11-24T22:14:28,202 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40087 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=63842f73863e9e7b9587486b122e6aae, server=bd6592c37bb2,40087,1732486423713 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-24T22:14:28,203 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40087 {}] ipc.CallRunner(138): callId: 32 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40226 deadline: 1732486528201, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=63842f73863e9e7b9587486b122e6aae, server=bd6592c37bb2,40087,1732486423713 2024-11-24T22:14:28,203 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732486460633.63842f73863e9e7b9587486b122e6aae., hostname=bd6592c37bb2,40087,1732486423713, seqNum=5 , the old value is region=TestAcidGuarantees,,1732486460633.63842f73863e9e7b9587486b122e6aae., hostname=bd6592c37bb2,40087,1732486423713, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=63842f73863e9e7b9587486b122e6aae, server=bd6592c37bb2,40087,1732486423713 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T22:14:28,203 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732486460633.63842f73863e9e7b9587486b122e6aae., hostname=bd6592c37bb2,40087,1732486423713, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=63842f73863e9e7b9587486b122e6aae, server=bd6592c37bb2,40087,1732486423713 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T22:14:28,203 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732486460633.63842f73863e9e7b9587486b122e6aae., hostname=bd6592c37bb2,40087,1732486423713, seqNum=5 because the exception is null or not the one we care about 2024-11-24T22:14:28,203 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40087 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=63842f73863e9e7b9587486b122e6aae, server=bd6592c37bb2,40087,1732486423713 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-24T22:14:28,204 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40087 {}] ipc.CallRunner(138): callId: 32 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40232 deadline: 1732486528201, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=63842f73863e9e7b9587486b122e6aae, server=bd6592c37bb2,40087,1732486423713 2024-11-24T22:14:28,204 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732486460633.63842f73863e9e7b9587486b122e6aae., hostname=bd6592c37bb2,40087,1732486423713, seqNum=5 , the old value is region=TestAcidGuarantees,,1732486460633.63842f73863e9e7b9587486b122e6aae., hostname=bd6592c37bb2,40087,1732486423713, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=63842f73863e9e7b9587486b122e6aae, server=bd6592c37bb2,40087,1732486423713 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T22:14:28,204 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732486460633.63842f73863e9e7b9587486b122e6aae., hostname=bd6592c37bb2,40087,1732486423713, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=63842f73863e9e7b9587486b122e6aae, server=bd6592c37bb2,40087,1732486423713 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T22:14:28,204 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732486460633.63842f73863e9e7b9587486b122e6aae., hostname=bd6592c37bb2,40087,1732486423713, seqNum=5 because the exception is null or not the one we care about 2024-11-24T22:14:28,209 INFO [RS:0;bd6592c37bb2:40087-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 63842f73863e9e7b9587486b122e6aae#A#compaction#118 average throughput is 0.19 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-24T22:14:28,210 DEBUG [RS:0;bd6592c37bb2:40087-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/63842f73863e9e7b9587486b122e6aae/.tmp/A/7a8e21a07dbe4b9f9a240d4f840c9c50 is 175, key is test_row_0/A:col10/1732486466601/Put/seqid=0 2024-11-24T22:14:28,218 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45929 is added to blk_1073741966_1142 (size=31161) 2024-11-24T22:14:28,231 DEBUG [RS:0;bd6592c37bb2:40087-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/63842f73863e9e7b9587486b122e6aae/.tmp/A/7a8e21a07dbe4b9f9a240d4f840c9c50 as hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/63842f73863e9e7b9587486b122e6aae/A/7a8e21a07dbe4b9f9a240d4f840c9c50 2024-11-24T22:14:28,241 INFO [RS:0;bd6592c37bb2:40087-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 63842f73863e9e7b9587486b122e6aae/A of 63842f73863e9e7b9587486b122e6aae into 7a8e21a07dbe4b9f9a240d4f840c9c50(size=30.4 K), total size for store is 30.4 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-24T22:14:28,241 DEBUG [RS:0;bd6592c37bb2:40087-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 63842f73863e9e7b9587486b122e6aae: 2024-11-24T22:14:28,241 INFO [RS:0;bd6592c37bb2:40087-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732486460633.63842f73863e9e7b9587486b122e6aae., storeName=63842f73863e9e7b9587486b122e6aae/A, priority=13, startTime=1732486468036; duration=0sec 2024-11-24T22:14:28,241 DEBUG [RS:0;bd6592c37bb2:40087-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-24T22:14:28,242 DEBUG [RS:0;bd6592c37bb2:40087-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 63842f73863e9e7b9587486b122e6aae:A 2024-11-24T22:14:28,242 DEBUG [RS:0;bd6592c37bb2:40087-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-24T22:14:28,243 DEBUG [RS:0;bd6592c37bb2:40087-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36106 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-24T22:14:28,243 DEBUG [RS:0;bd6592c37bb2:40087-shortCompactions-0 {}] regionserver.HStore(1541): 63842f73863e9e7b9587486b122e6aae/C is initiating minor compaction (all files) 2024-11-24T22:14:28,243 INFO [RS:0;bd6592c37bb2:40087-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 63842f73863e9e7b9587486b122e6aae/C in TestAcidGuarantees,,1732486460633.63842f73863e9e7b9587486b122e6aae. 2024-11-24T22:14:28,244 INFO [RS:0;bd6592c37bb2:40087-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/63842f73863e9e7b9587486b122e6aae/C/c3ebf3de6c6240cdb34d8d2cd189aadf, hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/63842f73863e9e7b9587486b122e6aae/C/6897f56dbe744713beb5afa1e154ea02, hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/63842f73863e9e7b9587486b122e6aae/C/17481ead002944a3b87ffd04ccafc446] into tmpdir=hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/63842f73863e9e7b9587486b122e6aae/.tmp, totalSize=35.3 K 2024-11-24T22:14:28,249 DEBUG [RS:0;bd6592c37bb2:40087-shortCompactions-0 {}] compactions.Compactor(225): Compacting c3ebf3de6c6240cdb34d8d2cd189aadf, keycount=150, bloomtype=ROW, size=11.8 K, encoding=NONE, compression=NONE, seqNum=65, earliestPutTs=1732486465173 2024-11-24T22:14:28,249 DEBUG [RS:0;bd6592c37bb2:40087-shortCompactions-0 {}] compactions.Compactor(225): Compacting 6897f56dbe744713beb5afa1e154ea02, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=85, earliestPutTs=1732486465856 2024-11-24T22:14:28,250 DEBUG [RS:0;bd6592c37bb2:40087-shortCompactions-0 {}] compactions.Compactor(225): Compacting 17481ead002944a3b87ffd04ccafc446, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=102, earliestPutTs=1732486466601 2024-11-24T22:14:28,274 INFO [RS:0;bd6592c37bb2:40087-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 63842f73863e9e7b9587486b122e6aae#C#compaction#120 average throughput is 1.64 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-24T22:14:28,274 DEBUG [RS:0;bd6592c37bb2:40087-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/63842f73863e9e7b9587486b122e6aae/.tmp/C/827f6af5e50f48edb4d996590552ae1d is 50, key is test_row_0/C:col10/1732486466601/Put/seqid=0 2024-11-24T22:14:28,285 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45929 is added to blk_1073741967_1143 (size=12207) 2024-11-24T22:14:28,298 DEBUG [RS:0;bd6592c37bb2:40087-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/63842f73863e9e7b9587486b122e6aae/.tmp/C/827f6af5e50f48edb4d996590552ae1d as hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/63842f73863e9e7b9587486b122e6aae/C/827f6af5e50f48edb4d996590552ae1d 2024-11-24T22:14:28,304 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=40087 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=41 2024-11-24T22:14:28,305 INFO [RS:0;bd6592c37bb2:40087-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 63842f73863e9e7b9587486b122e6aae/C of 63842f73863e9e7b9587486b122e6aae into 827f6af5e50f48edb4d996590552ae1d(size=11.9 K), total size for store is 11.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-24T22:14:28,305 DEBUG [RS:0;bd6592c37bb2:40087-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 63842f73863e9e7b9587486b122e6aae: 2024-11-24T22:14:28,305 INFO [RS:0;bd6592c37bb2:40087-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732486460633.63842f73863e9e7b9587486b122e6aae., storeName=63842f73863e9e7b9587486b122e6aae/C, priority=13, startTime=1732486468036; duration=0sec 2024-11-24T22:14:28,305 DEBUG [RS:0;bd6592c37bb2:40087-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-24T22:14:28,306 DEBUG [RS:0;bd6592c37bb2:40087-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 63842f73863e9e7b9587486b122e6aae:C 2024-11-24T22:14:28,306 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bd6592c37bb2:0-0 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732486460633.63842f73863e9e7b9587486b122e6aae. 2024-11-24T22:14:28,306 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bd6592c37bb2:0-0 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732486460633.63842f73863e9e7b9587486b122e6aae. as already flushing 2024-11-24T22:14:28,306 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bd6592c37bb2:0-0 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732486460633.63842f73863e9e7b9587486b122e6aae. 2024-11-24T22:14:28,306 ERROR [RS_FLUSH_OPERATIONS-regionserver/bd6592c37bb2:0-0 {event_type=RS_FLUSH_REGIONS, pid=41}] handler.RSProcedureHandler(58): pid=41 java.io.IOException: Unable to complete flush {ENCODED => 63842f73863e9e7b9587486b122e6aae, NAME => 'TestAcidGuarantees,,1732486460633.63842f73863e9e7b9587486b122e6aae.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T22:14:28,306 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bd6592c37bb2:0-0 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=41 java.io.IOException: Unable to complete flush {ENCODED => 63842f73863e9e7b9587486b122e6aae, NAME => 'TestAcidGuarantees,,1732486460633.63842f73863e9e7b9587486b122e6aae.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T22:14:28,307 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37701 {}] master.HMaster(4177): Remote procedure failed, pid=41 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 63842f73863e9e7b9587486b122e6aae, NAME => 'TestAcidGuarantees,,1732486460633.63842f73863e9e7b9587486b122e6aae.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 63842f73863e9e7b9587486b122e6aae, NAME => 'TestAcidGuarantees,,1732486460633.63842f73863e9e7b9587486b122e6aae.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T22:14:28,415 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40087 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=63842f73863e9e7b9587486b122e6aae, server=bd6592c37bb2,40087,1732486423713 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-24T22:14:28,415 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40087 {}] ipc.CallRunner(138): callId: 33 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40232 deadline: 1732486528411, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=63842f73863e9e7b9587486b122e6aae, server=bd6592c37bb2,40087,1732486423713 2024-11-24T22:14:28,416 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732486460633.63842f73863e9e7b9587486b122e6aae., hostname=bd6592c37bb2,40087,1732486423713, seqNum=5 , the old value is region=TestAcidGuarantees,,1732486460633.63842f73863e9e7b9587486b122e6aae., hostname=bd6592c37bb2,40087,1732486423713, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=63842f73863e9e7b9587486b122e6aae, server=bd6592c37bb2,40087,1732486423713 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T22:14:28,416 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40087 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=63842f73863e9e7b9587486b122e6aae, server=bd6592c37bb2,40087,1732486423713 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-24T22:14:28,416 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732486460633.63842f73863e9e7b9587486b122e6aae., hostname=bd6592c37bb2,40087,1732486423713, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=63842f73863e9e7b9587486b122e6aae, server=bd6592c37bb2,40087,1732486423713 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T22:14:28,416 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732486460633.63842f73863e9e7b9587486b122e6aae., hostname=bd6592c37bb2,40087,1732486423713, seqNum=5 because the exception is null or not the one we care about 2024-11-24T22:14:28,416 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40087 {}] ipc.CallRunner(138): callId: 33 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40226 deadline: 1732486528411, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=63842f73863e9e7b9587486b122e6aae, server=bd6592c37bb2,40087,1732486423713 2024-11-24T22:14:28,416 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732486460633.63842f73863e9e7b9587486b122e6aae., hostname=bd6592c37bb2,40087,1732486423713, seqNum=5 , the old value is region=TestAcidGuarantees,,1732486460633.63842f73863e9e7b9587486b122e6aae., hostname=bd6592c37bb2,40087,1732486423713, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=63842f73863e9e7b9587486b122e6aae, server=bd6592c37bb2,40087,1732486423713 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T22:14:28,416 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732486460633.63842f73863e9e7b9587486b122e6aae., hostname=bd6592c37bb2,40087,1732486423713, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=63842f73863e9e7b9587486b122e6aae, server=bd6592c37bb2,40087,1732486423713 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T22:14:28,417 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732486460633.63842f73863e9e7b9587486b122e6aae., hostname=bd6592c37bb2,40087,1732486423713, seqNum=5 because the exception is null or not the one we care about 2024-11-24T22:14:28,460 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=40087 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=41 2024-11-24T22:14:28,460 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bd6592c37bb2:0-1 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732486460633.63842f73863e9e7b9587486b122e6aae. 2024-11-24T22:14:28,461 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bd6592c37bb2:0-1 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732486460633.63842f73863e9e7b9587486b122e6aae. as already flushing 2024-11-24T22:14:28,461 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bd6592c37bb2:0-1 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732486460633.63842f73863e9e7b9587486b122e6aae. 2024-11-24T22:14:28,461 ERROR [RS_FLUSH_OPERATIONS-regionserver/bd6592c37bb2:0-1 {event_type=RS_FLUSH_REGIONS, pid=41}] handler.RSProcedureHandler(58): pid=41 java.io.IOException: Unable to complete flush {ENCODED => 63842f73863e9e7b9587486b122e6aae, NAME => 'TestAcidGuarantees,,1732486460633.63842f73863e9e7b9587486b122e6aae.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T22:14:28,461 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bd6592c37bb2:0-1 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=41 java.io.IOException: Unable to complete flush {ENCODED => 63842f73863e9e7b9587486b122e6aae, NAME => 'TestAcidGuarantees,,1732486460633.63842f73863e9e7b9587486b122e6aae.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T22:14:28,462 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37701 {}] master.HMaster(4177): Remote procedure failed, pid=41 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 63842f73863e9e7b9587486b122e6aae, NAME => 'TestAcidGuarantees,,1732486460633.63842f73863e9e7b9587486b122e6aae.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 63842f73863e9e7b9587486b122e6aae, NAME => 'TestAcidGuarantees,,1732486460633.63842f73863e9e7b9587486b122e6aae.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T22:14:28,473 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40087 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=63842f73863e9e7b9587486b122e6aae, server=bd6592c37bb2,40087,1732486423713 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-24T22:14:28,473 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40087 {}] ipc.CallRunner(138): callId: 49 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40206 deadline: 1732486528471, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=63842f73863e9e7b9587486b122e6aae, server=bd6592c37bb2,40087,1732486423713 2024-11-24T22:14:28,474 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40087 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=63842f73863e9e7b9587486b122e6aae, server=bd6592c37bb2,40087,1732486423713 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-24T22:14:28,474 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40087 {}] ipc.CallRunner(138): callId: 41 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40244 deadline: 1732486528471, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=63842f73863e9e7b9587486b122e6aae, server=bd6592c37bb2,40087,1732486423713 2024-11-24T22:14:28,474 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732486460633.63842f73863e9e7b9587486b122e6aae., hostname=bd6592c37bb2,40087,1732486423713, seqNum=5 , the old value is region=TestAcidGuarantees,,1732486460633.63842f73863e9e7b9587486b122e6aae., hostname=bd6592c37bb2,40087,1732486423713, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=63842f73863e9e7b9587486b122e6aae, server=bd6592c37bb2,40087,1732486423713 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T22:14:28,474 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732486460633.63842f73863e9e7b9587486b122e6aae., hostname=bd6592c37bb2,40087,1732486423713, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=63842f73863e9e7b9587486b122e6aae, server=bd6592c37bb2,40087,1732486423713 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T22:14:28,474 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732486460633.63842f73863e9e7b9587486b122e6aae., hostname=bd6592c37bb2,40087,1732486423713, seqNum=5 because the exception is null or not the one we care about 2024-11-24T22:14:28,474 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732486460633.63842f73863e9e7b9587486b122e6aae., hostname=bd6592c37bb2,40087,1732486423713, seqNum=5 , the old value is region=TestAcidGuarantees,,1732486460633.63842f73863e9e7b9587486b122e6aae., hostname=bd6592c37bb2,40087,1732486423713, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=63842f73863e9e7b9587486b122e6aae, server=bd6592c37bb2,40087,1732486423713 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T22:14:28,474 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732486460633.63842f73863e9e7b9587486b122e6aae., hostname=bd6592c37bb2,40087,1732486423713, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=63842f73863e9e7b9587486b122e6aae, server=bd6592c37bb2,40087,1732486423713 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T22:14:28,474 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732486460633.63842f73863e9e7b9587486b122e6aae., hostname=bd6592c37bb2,40087,1732486423713, seqNum=5 because the exception is null or not the one we care about 2024-11-24T22:14:28,563 DEBUG [RS:0;bd6592c37bb2:40087-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/63842f73863e9e7b9587486b122e6aae/.tmp/B/85d047d5e1164e14a0ecf07a70e8146c as hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/63842f73863e9e7b9587486b122e6aae/B/85d047d5e1164e14a0ecf07a70e8146c 2024-11-24T22:14:28,575 INFO [RS:0;bd6592c37bb2:40087-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 63842f73863e9e7b9587486b122e6aae/B of 63842f73863e9e7b9587486b122e6aae into 85d047d5e1164e14a0ecf07a70e8146c(size=11.9 K), total size for store is 11.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-24T22:14:28,575 DEBUG [RS:0;bd6592c37bb2:40087-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 63842f73863e9e7b9587486b122e6aae: 2024-11-24T22:14:28,575 INFO [RS:0;bd6592c37bb2:40087-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732486460633.63842f73863e9e7b9587486b122e6aae., storeName=63842f73863e9e7b9587486b122e6aae/B, priority=13, startTime=1732486468036; duration=0sec 2024-11-24T22:14:28,575 DEBUG [RS:0;bd6592c37bb2:40087-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-24T22:14:28,575 DEBUG [RS:0;bd6592c37bb2:40087-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 63842f73863e9e7b9587486b122e6aae:B 2024-11-24T22:14:28,599 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T22:14:28,611 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024112426339100592e4032b4171064d82de25b_63842f73863e9e7b9587486b122e6aae to hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024112426339100592e4032b4171064d82de25b_63842f73863e9e7b9587486b122e6aae 2024-11-24T22:14:28,617 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=40087 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=41 2024-11-24T22:14:28,617 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/63842f73863e9e7b9587486b122e6aae/.tmp/A/79994ec69119412bacc781ba6aa897bb, store: [table=TestAcidGuarantees family=A region=63842f73863e9e7b9587486b122e6aae] 2024-11-24T22:14:28,618 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bd6592c37bb2:0-2 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732486460633.63842f73863e9e7b9587486b122e6aae. 2024-11-24T22:14:28,618 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bd6592c37bb2:0-2 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732486460633.63842f73863e9e7b9587486b122e6aae. as already flushing 2024-11-24T22:14:28,618 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bd6592c37bb2:0-2 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732486460633.63842f73863e9e7b9587486b122e6aae. 2024-11-24T22:14:28,618 ERROR [RS_FLUSH_OPERATIONS-regionserver/bd6592c37bb2:0-2 {event_type=RS_FLUSH_REGIONS, pid=41}] handler.RSProcedureHandler(58): pid=41 java.io.IOException: Unable to complete flush {ENCODED => 63842f73863e9e7b9587486b122e6aae, NAME => 'TestAcidGuarantees,,1732486460633.63842f73863e9e7b9587486b122e6aae.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T22:14:28,618 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bd6592c37bb2:0-2 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=41 java.io.IOException: Unable to complete flush {ENCODED => 63842f73863e9e7b9587486b122e6aae, NAME => 'TestAcidGuarantees,,1732486460633.63842f73863e9e7b9587486b122e6aae.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T22:14:28,618 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/63842f73863e9e7b9587486b122e6aae/.tmp/A/79994ec69119412bacc781ba6aa897bb is 175, key is test_row_0/A:col10/1732486467804/Put/seqid=0 2024-11-24T22:14:28,619 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37701 {}] master.HMaster(4177): Remote procedure failed, pid=41 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 63842f73863e9e7b9587486b122e6aae, NAME => 'TestAcidGuarantees,,1732486460633.63842f73863e9e7b9587486b122e6aae.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 63842f73863e9e7b9587486b122e6aae, NAME => 'TestAcidGuarantees,,1732486460633.63842f73863e9e7b9587486b122e6aae.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T22:14:28,650 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45929 is added to blk_1073741968_1144 (size=30955) 2024-11-24T22:14:28,651 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=122, memsize=38.0 K, hasBloomFilter=true, into tmp file hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/63842f73863e9e7b9587486b122e6aae/.tmp/A/79994ec69119412bacc781ba6aa897bb 2024-11-24T22:14:28,672 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/63842f73863e9e7b9587486b122e6aae/.tmp/B/5eda402104654194b9e33e227dc03d50 is 50, key is test_row_0/B:col10/1732486467804/Put/seqid=0 2024-11-24T22:14:28,694 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45929 is added to blk_1073741969_1145 (size=12001) 2024-11-24T22:14:28,696 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=38.02 KB at sequenceid=122 (bloomFilter=true), to=hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/63842f73863e9e7b9587486b122e6aae/.tmp/B/5eda402104654194b9e33e227dc03d50 2024-11-24T22:14:28,731 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/63842f73863e9e7b9587486b122e6aae/.tmp/C/6942817daf0640c5b53d955de9391e7c is 50, key is test_row_0/C:col10/1732486467804/Put/seqid=0 2024-11-24T22:14:28,731 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40087 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=63842f73863e9e7b9587486b122e6aae, server=bd6592c37bb2,40087,1732486423713 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-24T22:14:28,731 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40087 {}] ipc.CallRunner(138): callId: 34 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40232 deadline: 1732486528721, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=63842f73863e9e7b9587486b122e6aae, server=bd6592c37bb2,40087,1732486423713 2024-11-24T22:14:28,732 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732486460633.63842f73863e9e7b9587486b122e6aae., hostname=bd6592c37bb2,40087,1732486423713, seqNum=5 , the old value is region=TestAcidGuarantees,,1732486460633.63842f73863e9e7b9587486b122e6aae., hostname=bd6592c37bb2,40087,1732486423713, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=63842f73863e9e7b9587486b122e6aae, server=bd6592c37bb2,40087,1732486423713 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T22:14:28,732 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732486460633.63842f73863e9e7b9587486b122e6aae., hostname=bd6592c37bb2,40087,1732486423713, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=63842f73863e9e7b9587486b122e6aae, server=bd6592c37bb2,40087,1732486423713 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T22:14:28,732 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732486460633.63842f73863e9e7b9587486b122e6aae., hostname=bd6592c37bb2,40087,1732486423713, seqNum=5 because the exception is null or not the one we care about 2024-11-24T22:14:28,732 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40087 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=63842f73863e9e7b9587486b122e6aae, server=bd6592c37bb2,40087,1732486423713 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-24T22:14:28,733 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40087 {}] ipc.CallRunner(138): callId: 34 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40226 deadline: 1732486528721, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=63842f73863e9e7b9587486b122e6aae, server=bd6592c37bb2,40087,1732486423713 2024-11-24T22:14:28,735 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732486460633.63842f73863e9e7b9587486b122e6aae., hostname=bd6592c37bb2,40087,1732486423713, seqNum=5 , the old value is region=TestAcidGuarantees,,1732486460633.63842f73863e9e7b9587486b122e6aae., hostname=bd6592c37bb2,40087,1732486423713, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=63842f73863e9e7b9587486b122e6aae, server=bd6592c37bb2,40087,1732486423713 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T22:14:28,735 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732486460633.63842f73863e9e7b9587486b122e6aae., hostname=bd6592c37bb2,40087,1732486423713, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=63842f73863e9e7b9587486b122e6aae, server=bd6592c37bb2,40087,1732486423713 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T22:14:28,735 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732486460633.63842f73863e9e7b9587486b122e6aae., hostname=bd6592c37bb2,40087,1732486423713, seqNum=5 because the exception is null or not the one we care about 2024-11-24T22:14:28,757 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45929 is added to blk_1073741970_1146 (size=12001) 2024-11-24T22:14:28,758 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=38.02 KB at sequenceid=122 (bloomFilter=true), to=hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/63842f73863e9e7b9587486b122e6aae/.tmp/C/6942817daf0640c5b53d955de9391e7c 2024-11-24T22:14:28,764 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/63842f73863e9e7b9587486b122e6aae/.tmp/A/79994ec69119412bacc781ba6aa897bb as hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/63842f73863e9e7b9587486b122e6aae/A/79994ec69119412bacc781ba6aa897bb 2024-11-24T22:14:28,770 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/63842f73863e9e7b9587486b122e6aae/A/79994ec69119412bacc781ba6aa897bb, entries=150, sequenceid=122, filesize=30.2 K 2024-11-24T22:14:28,771 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=40087 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=41 2024-11-24T22:14:28,771 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bd6592c37bb2:0-0 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732486460633.63842f73863e9e7b9587486b122e6aae. 2024-11-24T22:14:28,771 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bd6592c37bb2:0-0 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732486460633.63842f73863e9e7b9587486b122e6aae. as already flushing 2024-11-24T22:14:28,772 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bd6592c37bb2:0-0 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732486460633.63842f73863e9e7b9587486b122e6aae. 2024-11-24T22:14:28,772 ERROR [RS_FLUSH_OPERATIONS-regionserver/bd6592c37bb2:0-0 {event_type=RS_FLUSH_REGIONS, pid=41}] handler.RSProcedureHandler(58): pid=41 java.io.IOException: Unable to complete flush {ENCODED => 63842f73863e9e7b9587486b122e6aae, NAME => 'TestAcidGuarantees,,1732486460633.63842f73863e9e7b9587486b122e6aae.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T22:14:28,772 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bd6592c37bb2:0-0 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=41 java.io.IOException: Unable to complete flush {ENCODED => 63842f73863e9e7b9587486b122e6aae, NAME => 'TestAcidGuarantees,,1732486460633.63842f73863e9e7b9587486b122e6aae.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T22:14:28,773 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37701 {}] master.HMaster(4177): Remote procedure failed, pid=41 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 63842f73863e9e7b9587486b122e6aae, NAME => 'TestAcidGuarantees,,1732486460633.63842f73863e9e7b9587486b122e6aae.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 63842f73863e9e7b9587486b122e6aae, NAME => 'TestAcidGuarantees,,1732486460633.63842f73863e9e7b9587486b122e6aae.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T22:14:28,774 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/63842f73863e9e7b9587486b122e6aae/.tmp/B/5eda402104654194b9e33e227dc03d50 as hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/63842f73863e9e7b9587486b122e6aae/B/5eda402104654194b9e33e227dc03d50 2024-11-24T22:14:28,783 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/63842f73863e9e7b9587486b122e6aae/B/5eda402104654194b9e33e227dc03d50, entries=150, sequenceid=122, filesize=11.7 K 2024-11-24T22:14:28,785 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/63842f73863e9e7b9587486b122e6aae/.tmp/C/6942817daf0640c5b53d955de9391e7c as hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/63842f73863e9e7b9587486b122e6aae/C/6942817daf0640c5b53d955de9391e7c 2024-11-24T22:14:28,794 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/63842f73863e9e7b9587486b122e6aae/C/6942817daf0640c5b53d955de9391e7c, entries=150, sequenceid=122, filesize=11.7 K 2024-11-24T22:14:28,797 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~114.05 KB/116790, heapSize ~339.38 KB/347520, currentSize=60.38 KB/61830 for 63842f73863e9e7b9587486b122e6aae in 752ms, sequenceid=122, compaction requested=false 2024-11-24T22:14:28,798 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 63842f73863e9e7b9587486b122e6aae: 2024-11-24T22:14:28,925 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=40087 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=41 2024-11-24T22:14:28,926 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bd6592c37bb2:0-1 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732486460633.63842f73863e9e7b9587486b122e6aae. 2024-11-24T22:14:28,926 INFO [RS_FLUSH_OPERATIONS-regionserver/bd6592c37bb2:0-1 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.HRegion(2902): Flushing 63842f73863e9e7b9587486b122e6aae 3/3 column families, dataSize=60.38 KB heapSize=180.05 KB 2024-11-24T22:14:28,926 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bd6592c37bb2:0-1 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 63842f73863e9e7b9587486b122e6aae, store=A 2024-11-24T22:14:28,926 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bd6592c37bb2:0-1 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-24T22:14:28,926 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bd6592c37bb2:0-1 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 63842f73863e9e7b9587486b122e6aae, store=B 2024-11-24T22:14:28,926 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bd6592c37bb2:0-1 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-24T22:14:28,926 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bd6592c37bb2:0-1 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 63842f73863e9e7b9587486b122e6aae, store=C 2024-11-24T22:14:28,927 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bd6592c37bb2:0-1 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-24T22:14:28,949 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bd6592c37bb2:0-1 {event_type=RS_FLUSH_REGIONS, pid=41}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411243b9b5f14f2294190802454150d5ad535_63842f73863e9e7b9587486b122e6aae is 50, key is test_row_0/A:col10/1732486468093/Put/seqid=0 2024-11-24T22:14:28,987 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40087 {}] regionserver.HRegion(8855): Flush requested on 63842f73863e9e7b9587486b122e6aae 2024-11-24T22:14:28,987 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732486460633.63842f73863e9e7b9587486b122e6aae. as already flushing 2024-11-24T22:14:28,993 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45929 is added to blk_1073741971_1147 (size=12304) 2024-11-24T22:14:29,051 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40087 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=63842f73863e9e7b9587486b122e6aae, server=bd6592c37bb2,40087,1732486423713 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-24T22:14:29,052 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40087 {}] ipc.CallRunner(138): callId: 58 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40206 deadline: 1732486529045, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=63842f73863e9e7b9587486b122e6aae, server=bd6592c37bb2,40087,1732486423713 2024-11-24T22:14:29,052 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40087 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=63842f73863e9e7b9587486b122e6aae, server=bd6592c37bb2,40087,1732486423713 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-24T22:14:29,052 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40087 {}] ipc.CallRunner(138): callId: 51 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40244 deadline: 1732486529046, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=63842f73863e9e7b9587486b122e6aae, server=bd6592c37bb2,40087,1732486423713 2024-11-24T22:14:29,053 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732486460633.63842f73863e9e7b9587486b122e6aae., hostname=bd6592c37bb2,40087,1732486423713, seqNum=5 , the old value is region=TestAcidGuarantees,,1732486460633.63842f73863e9e7b9587486b122e6aae., hostname=bd6592c37bb2,40087,1732486423713, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=63842f73863e9e7b9587486b122e6aae, server=bd6592c37bb2,40087,1732486423713 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T22:14:29,053 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732486460633.63842f73863e9e7b9587486b122e6aae., hostname=bd6592c37bb2,40087,1732486423713, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=63842f73863e9e7b9587486b122e6aae, server=bd6592c37bb2,40087,1732486423713 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T22:14:29,053 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732486460633.63842f73863e9e7b9587486b122e6aae., hostname=bd6592c37bb2,40087,1732486423713, seqNum=5 because the exception is null or not the one we care about 2024-11-24T22:14:29,055 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732486460633.63842f73863e9e7b9587486b122e6aae., hostname=bd6592c37bb2,40087,1732486423713, seqNum=5 , the old value is region=TestAcidGuarantees,,1732486460633.63842f73863e9e7b9587486b122e6aae., hostname=bd6592c37bb2,40087,1732486423713, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=63842f73863e9e7b9587486b122e6aae, server=bd6592c37bb2,40087,1732486423713 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T22:14:29,055 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732486460633.63842f73863e9e7b9587486b122e6aae., hostname=bd6592c37bb2,40087,1732486423713, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=63842f73863e9e7b9587486b122e6aae, server=bd6592c37bb2,40087,1732486423713 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T22:14:29,055 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732486460633.63842f73863e9e7b9587486b122e6aae., hostname=bd6592c37bb2,40087,1732486423713, seqNum=5 because the exception is null or not the one we care about 2024-11-24T22:14:29,163 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40087 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=63842f73863e9e7b9587486b122e6aae, server=bd6592c37bb2,40087,1732486423713 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-24T22:14:29,163 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40087 {}] ipc.CallRunner(138): callId: 59 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40206 deadline: 1732486529160, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=63842f73863e9e7b9587486b122e6aae, server=bd6592c37bb2,40087,1732486423713 2024-11-24T22:14:29,164 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40087 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=63842f73863e9e7b9587486b122e6aae, server=bd6592c37bb2,40087,1732486423713 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-24T22:14:29,164 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732486460633.63842f73863e9e7b9587486b122e6aae., hostname=bd6592c37bb2,40087,1732486423713, seqNum=5 , the old value is region=TestAcidGuarantees,,1732486460633.63842f73863e9e7b9587486b122e6aae., hostname=bd6592c37bb2,40087,1732486423713, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=63842f73863e9e7b9587486b122e6aae, server=bd6592c37bb2,40087,1732486423713 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T22:14:29,164 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732486460633.63842f73863e9e7b9587486b122e6aae., hostname=bd6592c37bb2,40087,1732486423713, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=63842f73863e9e7b9587486b122e6aae, server=bd6592c37bb2,40087,1732486423713 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T22:14:29,164 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40087 {}] ipc.CallRunner(138): callId: 52 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40244 deadline: 1732486529161, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=63842f73863e9e7b9587486b122e6aae, server=bd6592c37bb2,40087,1732486423713 2024-11-24T22:14:29,164 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732486460633.63842f73863e9e7b9587486b122e6aae., hostname=bd6592c37bb2,40087,1732486423713, seqNum=5 because the exception is null or not the one we care about 2024-11-24T22:14:29,164 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732486460633.63842f73863e9e7b9587486b122e6aae., hostname=bd6592c37bb2,40087,1732486423713, seqNum=5 , the old value is region=TestAcidGuarantees,,1732486460633.63842f73863e9e7b9587486b122e6aae., hostname=bd6592c37bb2,40087,1732486423713, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=63842f73863e9e7b9587486b122e6aae, server=bd6592c37bb2,40087,1732486423713 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T22:14:29,164 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732486460633.63842f73863e9e7b9587486b122e6aae., hostname=bd6592c37bb2,40087,1732486423713, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=63842f73863e9e7b9587486b122e6aae, server=bd6592c37bb2,40087,1732486423713 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T22:14:29,164 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732486460633.63842f73863e9e7b9587486b122e6aae., hostname=bd6592c37bb2,40087,1732486423713, seqNum=5 because the exception is null or not the one we care about 2024-11-24T22:14:29,246 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40087 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=63842f73863e9e7b9587486b122e6aae, server=bd6592c37bb2,40087,1732486423713 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-24T22:14:29,246 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40087 {}] ipc.CallRunner(138): callId: 35 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40232 deadline: 1732486529240, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=63842f73863e9e7b9587486b122e6aae, server=bd6592c37bb2,40087,1732486423713 2024-11-24T22:14:29,246 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40087 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=63842f73863e9e7b9587486b122e6aae, server=bd6592c37bb2,40087,1732486423713 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-24T22:14:29,246 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732486460633.63842f73863e9e7b9587486b122e6aae., hostname=bd6592c37bb2,40087,1732486423713, seqNum=5 , the old value is region=TestAcidGuarantees,,1732486460633.63842f73863e9e7b9587486b122e6aae., hostname=bd6592c37bb2,40087,1732486423713, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=63842f73863e9e7b9587486b122e6aae, server=bd6592c37bb2,40087,1732486423713 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T22:14:29,246 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732486460633.63842f73863e9e7b9587486b122e6aae., hostname=bd6592c37bb2,40087,1732486423713, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=63842f73863e9e7b9587486b122e6aae, server=bd6592c37bb2,40087,1732486423713 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T22:14:29,246 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40087 {}] ipc.CallRunner(138): callId: 35 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40226 deadline: 1732486529240, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=63842f73863e9e7b9587486b122e6aae, server=bd6592c37bb2,40087,1732486423713 2024-11-24T22:14:29,246 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732486460633.63842f73863e9e7b9587486b122e6aae., hostname=bd6592c37bb2,40087,1732486423713, seqNum=5 because the exception is null or not the one we care about 2024-11-24T22:14:29,247 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732486460633.63842f73863e9e7b9587486b122e6aae., hostname=bd6592c37bb2,40087,1732486423713, seqNum=5 , the old value is region=TestAcidGuarantees,,1732486460633.63842f73863e9e7b9587486b122e6aae., hostname=bd6592c37bb2,40087,1732486423713, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=63842f73863e9e7b9587486b122e6aae, server=bd6592c37bb2,40087,1732486423713 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T22:14:29,247 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732486460633.63842f73863e9e7b9587486b122e6aae., hostname=bd6592c37bb2,40087,1732486423713, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=63842f73863e9e7b9587486b122e6aae, server=bd6592c37bb2,40087,1732486423713 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T22:14:29,247 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732486460633.63842f73863e9e7b9587486b122e6aae., hostname=bd6592c37bb2,40087,1732486423713, seqNum=5 because the exception is null or not the one we care about 2024-11-24T22:14:29,373 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40087 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=63842f73863e9e7b9587486b122e6aae, server=bd6592c37bb2,40087,1732486423713 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-24T22:14:29,374 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40087 {}] ipc.CallRunner(138): callId: 60 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40206 deadline: 1732486529371, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=63842f73863e9e7b9587486b122e6aae, server=bd6592c37bb2,40087,1732486423713 2024-11-24T22:14:29,374 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40087 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=63842f73863e9e7b9587486b122e6aae, server=bd6592c37bb2,40087,1732486423713 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-24T22:14:29,374 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732486460633.63842f73863e9e7b9587486b122e6aae., hostname=bd6592c37bb2,40087,1732486423713, seqNum=5 , the old value is region=TestAcidGuarantees,,1732486460633.63842f73863e9e7b9587486b122e6aae., hostname=bd6592c37bb2,40087,1732486423713, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=63842f73863e9e7b9587486b122e6aae, server=bd6592c37bb2,40087,1732486423713 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T22:14:29,374 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732486460633.63842f73863e9e7b9587486b122e6aae., hostname=bd6592c37bb2,40087,1732486423713, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=63842f73863e9e7b9587486b122e6aae, server=bd6592c37bb2,40087,1732486423713 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T22:14:29,374 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40087 {}] ipc.CallRunner(138): callId: 53 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40244 deadline: 1732486529371, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=63842f73863e9e7b9587486b122e6aae, server=bd6592c37bb2,40087,1732486423713 2024-11-24T22:14:29,374 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732486460633.63842f73863e9e7b9587486b122e6aae., hostname=bd6592c37bb2,40087,1732486423713, seqNum=5 because the exception is null or not the one we care about 2024-11-24T22:14:29,374 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732486460633.63842f73863e9e7b9587486b122e6aae., hostname=bd6592c37bb2,40087,1732486423713, seqNum=5 , the old value is region=TestAcidGuarantees,,1732486460633.63842f73863e9e7b9587486b122e6aae., hostname=bd6592c37bb2,40087,1732486423713, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=63842f73863e9e7b9587486b122e6aae, server=bd6592c37bb2,40087,1732486423713 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T22:14:29,375 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732486460633.63842f73863e9e7b9587486b122e6aae., hostname=bd6592c37bb2,40087,1732486423713, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=63842f73863e9e7b9587486b122e6aae, server=bd6592c37bb2,40087,1732486423713 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T22:14:29,375 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732486460633.63842f73863e9e7b9587486b122e6aae., hostname=bd6592c37bb2,40087,1732486423713, seqNum=5 because the exception is null or not the one we care about 2024-11-24T22:14:29,394 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bd6592c37bb2:0-1 {event_type=RS_FLUSH_REGIONS, pid=41}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T22:14:29,399 INFO [RS_FLUSH_OPERATIONS-regionserver/bd6592c37bb2:0-1 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411243b9b5f14f2294190802454150d5ad535_63842f73863e9e7b9587486b122e6aae to hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411243b9b5f14f2294190802454150d5ad535_63842f73863e9e7b9587486b122e6aae 2024-11-24T22:14:29,404 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bd6592c37bb2:0-1 {event_type=RS_FLUSH_REGIONS, pid=41}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/63842f73863e9e7b9587486b122e6aae/.tmp/A/35f72322d95641559d1e43da1cc0a431, store: [table=TestAcidGuarantees family=A region=63842f73863e9e7b9587486b122e6aae] 2024-11-24T22:14:29,405 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bd6592c37bb2:0-1 {event_type=RS_FLUSH_REGIONS, pid=41}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/63842f73863e9e7b9587486b122e6aae/.tmp/A/35f72322d95641559d1e43da1cc0a431 is 175, key is test_row_0/A:col10/1732486468093/Put/seqid=0 2024-11-24T22:14:29,412 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45929 is added to blk_1073741972_1148 (size=31105) 2024-11-24T22:14:29,413 INFO [RS_FLUSH_OPERATIONS-regionserver/bd6592c37bb2:0-1 {event_type=RS_FLUSH_REGIONS, pid=41}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=137, memsize=20.1 K, hasBloomFilter=true, into tmp file hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/63842f73863e9e7b9587486b122e6aae/.tmp/A/35f72322d95641559d1e43da1cc0a431 2024-11-24T22:14:29,430 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bd6592c37bb2:0-1 {event_type=RS_FLUSH_REGIONS, pid=41}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/63842f73863e9e7b9587486b122e6aae/.tmp/B/7b7482699d6743d1874a0762ec11ae0e is 50, key is test_row_0/B:col10/1732486468093/Put/seqid=0 2024-11-24T22:14:29,446 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45929 is added to blk_1073741973_1149 (size=12151) 2024-11-24T22:14:29,681 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40087 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=63842f73863e9e7b9587486b122e6aae, server=bd6592c37bb2,40087,1732486423713 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-24T22:14:29,681 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40087 {}] ipc.CallRunner(138): callId: 54 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40244 deadline: 1732486529681, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=63842f73863e9e7b9587486b122e6aae, server=bd6592c37bb2,40087,1732486423713 2024-11-24T22:14:29,682 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40087 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=63842f73863e9e7b9587486b122e6aae, server=bd6592c37bb2,40087,1732486423713 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-24T22:14:29,682 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732486460633.63842f73863e9e7b9587486b122e6aae., hostname=bd6592c37bb2,40087,1732486423713, seqNum=5 , the old value is region=TestAcidGuarantees,,1732486460633.63842f73863e9e7b9587486b122e6aae., hostname=bd6592c37bb2,40087,1732486423713, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=63842f73863e9e7b9587486b122e6aae, server=bd6592c37bb2,40087,1732486423713 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T22:14:29,682 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732486460633.63842f73863e9e7b9587486b122e6aae., hostname=bd6592c37bb2,40087,1732486423713, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=63842f73863e9e7b9587486b122e6aae, server=bd6592c37bb2,40087,1732486423713 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T22:14:29,682 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40087 {}] ipc.CallRunner(138): callId: 61 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40206 deadline: 1732486529681, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=63842f73863e9e7b9587486b122e6aae, server=bd6592c37bb2,40087,1732486423713 2024-11-24T22:14:29,682 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732486460633.63842f73863e9e7b9587486b122e6aae., hostname=bd6592c37bb2,40087,1732486423713, seqNum=5 because the exception is null or not the one we care about 2024-11-24T22:14:29,682 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732486460633.63842f73863e9e7b9587486b122e6aae., hostname=bd6592c37bb2,40087,1732486423713, seqNum=5 , the old value is region=TestAcidGuarantees,,1732486460633.63842f73863e9e7b9587486b122e6aae., hostname=bd6592c37bb2,40087,1732486423713, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=63842f73863e9e7b9587486b122e6aae, server=bd6592c37bb2,40087,1732486423713 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T22:14:29,682 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732486460633.63842f73863e9e7b9587486b122e6aae., hostname=bd6592c37bb2,40087,1732486423713, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=63842f73863e9e7b9587486b122e6aae, server=bd6592c37bb2,40087,1732486423713 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T22:14:29,682 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732486460633.63842f73863e9e7b9587486b122e6aae., hostname=bd6592c37bb2,40087,1732486423713, seqNum=5 because the exception is null or not the one we care about 2024-11-24T22:14:29,847 INFO [RS_FLUSH_OPERATIONS-regionserver/bd6592c37bb2:0-1 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=137 (bloomFilter=true), to=hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/63842f73863e9e7b9587486b122e6aae/.tmp/B/7b7482699d6743d1874a0762ec11ae0e 2024-11-24T22:14:29,860 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bd6592c37bb2:0-1 {event_type=RS_FLUSH_REGIONS, pid=41}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/63842f73863e9e7b9587486b122e6aae/.tmp/C/4d52a0c91d794d46956f4bd1629386f4 is 50, key is test_row_0/C:col10/1732486468093/Put/seqid=0 2024-11-24T22:14:29,878 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45929 is added to blk_1073741974_1150 (size=12151) 2024-11-24T22:14:29,878 INFO [RS_FLUSH_OPERATIONS-regionserver/bd6592c37bb2:0-1 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=137 (bloomFilter=true), to=hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/63842f73863e9e7b9587486b122e6aae/.tmp/C/4d52a0c91d794d46956f4bd1629386f4 2024-11-24T22:14:29,890 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bd6592c37bb2:0-1 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/63842f73863e9e7b9587486b122e6aae/.tmp/A/35f72322d95641559d1e43da1cc0a431 as hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/63842f73863e9e7b9587486b122e6aae/A/35f72322d95641559d1e43da1cc0a431 2024-11-24T22:14:29,903 INFO [RS_FLUSH_OPERATIONS-regionserver/bd6592c37bb2:0-1 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/63842f73863e9e7b9587486b122e6aae/A/35f72322d95641559d1e43da1cc0a431, entries=150, sequenceid=137, filesize=30.4 K 2024-11-24T22:14:29,906 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bd6592c37bb2:0-1 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/63842f73863e9e7b9587486b122e6aae/.tmp/B/7b7482699d6743d1874a0762ec11ae0e as hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/63842f73863e9e7b9587486b122e6aae/B/7b7482699d6743d1874a0762ec11ae0e 2024-11-24T22:14:29,911 INFO [RS_FLUSH_OPERATIONS-regionserver/bd6592c37bb2:0-1 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/63842f73863e9e7b9587486b122e6aae/B/7b7482699d6743d1874a0762ec11ae0e, entries=150, sequenceid=137, filesize=11.9 K 2024-11-24T22:14:29,912 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bd6592c37bb2:0-1 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/63842f73863e9e7b9587486b122e6aae/.tmp/C/4d52a0c91d794d46956f4bd1629386f4 as hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/63842f73863e9e7b9587486b122e6aae/C/4d52a0c91d794d46956f4bd1629386f4 2024-11-24T22:14:29,919 INFO [RS_FLUSH_OPERATIONS-regionserver/bd6592c37bb2:0-1 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/63842f73863e9e7b9587486b122e6aae/C/4d52a0c91d794d46956f4bd1629386f4, entries=150, sequenceid=137, filesize=11.9 K 2024-11-24T22:14:29,920 INFO [RS_FLUSH_OPERATIONS-regionserver/bd6592c37bb2:0-1 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.HRegion(3140): Finished flush of dataSize ~60.38 KB/61830, heapSize ~180 KB/184320, currentSize=114.05 KB/116790 for 63842f73863e9e7b9587486b122e6aae in 994ms, sequenceid=137, compaction requested=true 2024-11-24T22:14:29,921 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bd6592c37bb2:0-1 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.HRegion(2603): Flush status journal for 63842f73863e9e7b9587486b122e6aae: 2024-11-24T22:14:29,921 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bd6592c37bb2:0-1 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732486460633.63842f73863e9e7b9587486b122e6aae. 2024-11-24T22:14:29,921 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bd6592c37bb2:0-1 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=41 2024-11-24T22:14:29,921 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37701 {}] master.HMaster(4169): Remote procedure done, pid=41 2024-11-24T22:14:29,926 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=41, resume processing ppid=40 2024-11-24T22:14:29,926 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=41, ppid=40, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 3.9550 sec 2024-11-24T22:14:29,928 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=40, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=40, table=TestAcidGuarantees in 3.9650 sec 2024-11-24T22:14:30,046 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 63842f73863e9e7b9587486b122e6aae 3/3 column families, dataSize=120.76 KB heapSize=360.05 KB 2024-11-24T22:14:30,046 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 63842f73863e9e7b9587486b122e6aae, store=A 2024-11-24T22:14:30,046 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-24T22:14:30,046 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 63842f73863e9e7b9587486b122e6aae, store=B 2024-11-24T22:14:30,047 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40087 {}] regionserver.HRegion(8855): Flush requested on 63842f73863e9e7b9587486b122e6aae 2024-11-24T22:14:30,048 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/bd6592c37bb2:0-1 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1732486469034, max timestamp=1732486470044Num uniques 859; 2024-11-24T22:14:30,048 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/bd6592c37bb2:0-1 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-11-24T22:14:30,047 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-24T22:14:30,049 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 63842f73863e9e7b9587486b122e6aae, store=C 2024-11-24T22:14:30,049 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-24T22:14:30,064 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241124fb2e73668a25415b989fcc090c3ce251_63842f73863e9e7b9587486b122e6aae is 50, key is test_row_0/A:col10/1732486470044/Put/seqid=0 2024-11-24T22:14:30,070 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45929 is added to blk_1073741975_1151 (size=14794) 2024-11-24T22:14:30,121 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40087 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=63842f73863e9e7b9587486b122e6aae, server=bd6592c37bb2,40087,1732486423713 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-24T22:14:30,121 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40087 {}] ipc.CallRunner(138): callId: 44 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40202 deadline: 1732486530114, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=63842f73863e9e7b9587486b122e6aae, server=bd6592c37bb2,40087,1732486423713 2024-11-24T22:14:30,122 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732486460633.63842f73863e9e7b9587486b122e6aae., hostname=bd6592c37bb2,40087,1732486423713, seqNum=5 , the old value is region=TestAcidGuarantees,,1732486460633.63842f73863e9e7b9587486b122e6aae., hostname=bd6592c37bb2,40087,1732486423713, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=63842f73863e9e7b9587486b122e6aae, server=bd6592c37bb2,40087,1732486423713 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T22:14:30,122 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732486460633.63842f73863e9e7b9587486b122e6aae., hostname=bd6592c37bb2,40087,1732486423713, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=63842f73863e9e7b9587486b122e6aae, server=bd6592c37bb2,40087,1732486423713 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T22:14:30,122 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732486460633.63842f73863e9e7b9587486b122e6aae., hostname=bd6592c37bb2,40087,1732486423713, seqNum=5 because the exception is null or not the one we care about 2024-11-24T22:14:30,131 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37701 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=40 2024-11-24T22:14:30,132 INFO [RPCClient-NioEventLoopGroup-4-7 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-24T22:14:30,132 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37701 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-24T22:14:30,133 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37701 {}] procedure2.ProcedureExecutor(1139): Stored pid=42, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=42, table=TestAcidGuarantees 2024-11-24T22:14:30,134 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=42, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=42, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-24T22:14:30,135 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=42, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=42, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-24T22:14:30,135 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=43, ppid=42, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-24T22:14:30,136 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37701 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=42 2024-11-24T22:14:30,197 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40087 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=63842f73863e9e7b9587486b122e6aae, server=bd6592c37bb2,40087,1732486423713 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-24T22:14:30,197 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40087 {}] ipc.CallRunner(138): callId: 62 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40206 deadline: 1732486530191, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=63842f73863e9e7b9587486b122e6aae, server=bd6592c37bb2,40087,1732486423713 2024-11-24T22:14:30,197 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40087 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=63842f73863e9e7b9587486b122e6aae, server=bd6592c37bb2,40087,1732486423713 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-24T22:14:30,198 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732486460633.63842f73863e9e7b9587486b122e6aae., hostname=bd6592c37bb2,40087,1732486423713, seqNum=5 , the old value is region=TestAcidGuarantees,,1732486460633.63842f73863e9e7b9587486b122e6aae., hostname=bd6592c37bb2,40087,1732486423713, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=63842f73863e9e7b9587486b122e6aae, server=bd6592c37bb2,40087,1732486423713 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T22:14:30,198 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732486460633.63842f73863e9e7b9587486b122e6aae., hostname=bd6592c37bb2,40087,1732486423713, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=63842f73863e9e7b9587486b122e6aae, server=bd6592c37bb2,40087,1732486423713 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T22:14:30,198 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732486460633.63842f73863e9e7b9587486b122e6aae., hostname=bd6592c37bb2,40087,1732486423713, seqNum=5 because the exception is null or not the one we care about 2024-11-24T22:14:30,198 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40087 {}] ipc.CallRunner(138): callId: 55 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40244 deadline: 1732486530191, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=63842f73863e9e7b9587486b122e6aae, server=bd6592c37bb2,40087,1732486423713 2024-11-24T22:14:30,198 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732486460633.63842f73863e9e7b9587486b122e6aae., hostname=bd6592c37bb2,40087,1732486423713, seqNum=5 , the old value is region=TestAcidGuarantees,,1732486460633.63842f73863e9e7b9587486b122e6aae., hostname=bd6592c37bb2,40087,1732486423713, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=63842f73863e9e7b9587486b122e6aae, server=bd6592c37bb2,40087,1732486423713 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T22:14:30,198 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732486460633.63842f73863e9e7b9587486b122e6aae., hostname=bd6592c37bb2,40087,1732486423713, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=63842f73863e9e7b9587486b122e6aae, server=bd6592c37bb2,40087,1732486423713 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T22:14:30,198 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732486460633.63842f73863e9e7b9587486b122e6aae., hostname=bd6592c37bb2,40087,1732486423713, seqNum=5 because the exception is null or not the one we care about 2024-11-24T22:14:30,235 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40087 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=63842f73863e9e7b9587486b122e6aae, server=bd6592c37bb2,40087,1732486423713 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-24T22:14:30,235 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40087 {}] ipc.CallRunner(138): callId: 45 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40202 deadline: 1732486530231, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=63842f73863e9e7b9587486b122e6aae, server=bd6592c37bb2,40087,1732486423713 2024-11-24T22:14:30,235 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732486460633.63842f73863e9e7b9587486b122e6aae., hostname=bd6592c37bb2,40087,1732486423713, seqNum=5 , the old value is region=TestAcidGuarantees,,1732486460633.63842f73863e9e7b9587486b122e6aae., hostname=bd6592c37bb2,40087,1732486423713, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=63842f73863e9e7b9587486b122e6aae, server=bd6592c37bb2,40087,1732486423713 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T22:14:30,235 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732486460633.63842f73863e9e7b9587486b122e6aae., hostname=bd6592c37bb2,40087,1732486423713, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=63842f73863e9e7b9587486b122e6aae, server=bd6592c37bb2,40087,1732486423713 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T22:14:30,235 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732486460633.63842f73863e9e7b9587486b122e6aae., hostname=bd6592c37bb2,40087,1732486423713, seqNum=5 because the exception is null or not the one we care about 2024-11-24T22:14:30,241 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37701 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=42 2024-11-24T22:14:30,263 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40087 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=63842f73863e9e7b9587486b122e6aae, server=bd6592c37bb2,40087,1732486423713 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-24T22:14:30,264 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40087 {}] ipc.CallRunner(138): callId: 36 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40232 deadline: 1732486530260, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=63842f73863e9e7b9587486b122e6aae, server=bd6592c37bb2,40087,1732486423713 2024-11-24T22:14:30,264 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40087 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=63842f73863e9e7b9587486b122e6aae, server=bd6592c37bb2,40087,1732486423713 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-24T22:14:30,264 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40087 {}] ipc.CallRunner(138): callId: 36 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40226 deadline: 1732486530261, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=63842f73863e9e7b9587486b122e6aae, server=bd6592c37bb2,40087,1732486423713 2024-11-24T22:14:30,264 WARN [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRpcRetryingCaller(177): Call to bd6592c37bb2,40087,1732486423713 for 'test_row_1' in 63842f73863e9e7b9587486b122e6aae of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2169 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=63842f73863e9e7b9587486b122e6aae, server=bd6592c37bb2,40087,1732486423713 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor44.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=63842f73863e9e7b9587486b122e6aae, server=bd6592c37bb2,40087,1732486423713 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-24T22:14:30,265 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732486460633.63842f73863e9e7b9587486b122e6aae., hostname=bd6592c37bb2,40087,1732486423713, seqNum=5 , the old value is region=TestAcidGuarantees,,1732486460633.63842f73863e9e7b9587486b122e6aae., hostname=bd6592c37bb2,40087,1732486423713, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=63842f73863e9e7b9587486b122e6aae, server=bd6592c37bb2,40087,1732486423713 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T22:14:30,265 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732486460633.63842f73863e9e7b9587486b122e6aae., hostname=bd6592c37bb2,40087,1732486423713, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=63842f73863e9e7b9587486b122e6aae, server=bd6592c37bb2,40087,1732486423713 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T22:14:30,265 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732486460633.63842f73863e9e7b9587486b122e6aae., hostname=bd6592c37bb2,40087,1732486423713, seqNum=5 because the exception is null or not the one we care about 2024-11-24T22:14:30,265 WARN [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRpcRetryingCaller(177): Call to bd6592c37bb2,40087,1732486423713 for 'test_row_2' in 63842f73863e9e7b9587486b122e6aae of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2176 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=63842f73863e9e7b9587486b122e6aae, server=bd6592c37bb2,40087,1732486423713 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor44.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=63842f73863e9e7b9587486b122e6aae, server=bd6592c37bb2,40087,1732486423713 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-24T22:14:30,265 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732486460633.63842f73863e9e7b9587486b122e6aae., hostname=bd6592c37bb2,40087,1732486423713, seqNum=5 , the old value is region=TestAcidGuarantees,,1732486460633.63842f73863e9e7b9587486b122e6aae., hostname=bd6592c37bb2,40087,1732486423713, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=63842f73863e9e7b9587486b122e6aae, server=bd6592c37bb2,40087,1732486423713 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T22:14:30,265 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732486460633.63842f73863e9e7b9587486b122e6aae., hostname=bd6592c37bb2,40087,1732486423713, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=63842f73863e9e7b9587486b122e6aae, server=bd6592c37bb2,40087,1732486423713 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T22:14:30,265 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732486460633.63842f73863e9e7b9587486b122e6aae., hostname=bd6592c37bb2,40087,1732486423713, seqNum=5 because the exception is null or not the one we care about 2024-11-24T22:14:30,291 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=40087 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=43 2024-11-24T22:14:30,291 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bd6592c37bb2:0-2 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732486460633.63842f73863e9e7b9587486b122e6aae. 2024-11-24T22:14:30,291 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bd6592c37bb2:0-2 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732486460633.63842f73863e9e7b9587486b122e6aae. as already flushing 2024-11-24T22:14:30,291 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bd6592c37bb2:0-2 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732486460633.63842f73863e9e7b9587486b122e6aae. 2024-11-24T22:14:30,291 ERROR [RS_FLUSH_OPERATIONS-regionserver/bd6592c37bb2:0-2 {event_type=RS_FLUSH_REGIONS, pid=43}] handler.RSProcedureHandler(58): pid=43 java.io.IOException: Unable to complete flush {ENCODED => 63842f73863e9e7b9587486b122e6aae, NAME => 'TestAcidGuarantees,,1732486460633.63842f73863e9e7b9587486b122e6aae.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T22:14:30,292 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bd6592c37bb2:0-2 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=43 java.io.IOException: Unable to complete flush {ENCODED => 63842f73863e9e7b9587486b122e6aae, NAME => 'TestAcidGuarantees,,1732486460633.63842f73863e9e7b9587486b122e6aae.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T22:14:30,292 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37701 {}] master.HMaster(4177): Remote procedure failed, pid=43 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 63842f73863e9e7b9587486b122e6aae, NAME => 'TestAcidGuarantees,,1732486460633.63842f73863e9e7b9587486b122e6aae.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 63842f73863e9e7b9587486b122e6aae, NAME => 'TestAcidGuarantees,,1732486460633.63842f73863e9e7b9587486b122e6aae.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T22:14:30,443 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40087 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=63842f73863e9e7b9587486b122e6aae, server=bd6592c37bb2,40087,1732486423713 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-24T22:14:30,443 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40087 {}] ipc.CallRunner(138): callId: 46 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40202 deadline: 1732486530441, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=63842f73863e9e7b9587486b122e6aae, server=bd6592c37bb2,40087,1732486423713 2024-11-24T22:14:30,444 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732486460633.63842f73863e9e7b9587486b122e6aae., hostname=bd6592c37bb2,40087,1732486423713, seqNum=5 , the old value is region=TestAcidGuarantees,,1732486460633.63842f73863e9e7b9587486b122e6aae., hostname=bd6592c37bb2,40087,1732486423713, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=63842f73863e9e7b9587486b122e6aae, server=bd6592c37bb2,40087,1732486423713 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T22:14:30,444 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732486460633.63842f73863e9e7b9587486b122e6aae., hostname=bd6592c37bb2,40087,1732486423713, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=63842f73863e9e7b9587486b122e6aae, server=bd6592c37bb2,40087,1732486423713 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T22:14:30,444 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732486460633.63842f73863e9e7b9587486b122e6aae., hostname=bd6592c37bb2,40087,1732486423713, seqNum=5 because the exception is null or not the one we care about 2024-11-24T22:14:30,445 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=40087 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=43 2024-11-24T22:14:30,445 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bd6592c37bb2:0-0 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732486460633.63842f73863e9e7b9587486b122e6aae. 2024-11-24T22:14:30,446 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bd6592c37bb2:0-0 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732486460633.63842f73863e9e7b9587486b122e6aae. as already flushing 2024-11-24T22:14:30,446 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bd6592c37bb2:0-0 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732486460633.63842f73863e9e7b9587486b122e6aae. 2024-11-24T22:14:30,446 ERROR [RS_FLUSH_OPERATIONS-regionserver/bd6592c37bb2:0-0 {event_type=RS_FLUSH_REGIONS, pid=43}] handler.RSProcedureHandler(58): pid=43 java.io.IOException: Unable to complete flush {ENCODED => 63842f73863e9e7b9587486b122e6aae, NAME => 'TestAcidGuarantees,,1732486460633.63842f73863e9e7b9587486b122e6aae.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T22:14:30,446 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bd6592c37bb2:0-0 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=43 java.io.IOException: Unable to complete flush {ENCODED => 63842f73863e9e7b9587486b122e6aae, NAME => 'TestAcidGuarantees,,1732486460633.63842f73863e9e7b9587486b122e6aae.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T22:14:30,449 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37701 {}] master.HMaster(4177): Remote procedure failed, pid=43 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 63842f73863e9e7b9587486b122e6aae, NAME => 'TestAcidGuarantees,,1732486460633.63842f73863e9e7b9587486b122e6aae.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 63842f73863e9e7b9587486b122e6aae, NAME => 'TestAcidGuarantees,,1732486460633.63842f73863e9e7b9587486b122e6aae.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T22:14:30,450 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37701 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=42 2024-11-24T22:14:30,471 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T22:14:30,476 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241124fb2e73668a25415b989fcc090c3ce251_63842f73863e9e7b9587486b122e6aae to hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241124fb2e73668a25415b989fcc090c3ce251_63842f73863e9e7b9587486b122e6aae 2024-11-24T22:14:30,477 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/63842f73863e9e7b9587486b122e6aae/.tmp/A/a9d13b63e2ed4a508c48dec40823e1c0, store: [table=TestAcidGuarantees family=A region=63842f73863e9e7b9587486b122e6aae] 2024-11-24T22:14:30,478 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/63842f73863e9e7b9587486b122e6aae/.tmp/A/a9d13b63e2ed4a508c48dec40823e1c0 is 175, key is test_row_0/A:col10/1732486470044/Put/seqid=0 2024-11-24T22:14:30,486 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45929 is added to blk_1073741976_1152 (size=39749) 2024-11-24T22:14:30,601 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=40087 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=43 2024-11-24T22:14:30,601 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bd6592c37bb2:0-1 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732486460633.63842f73863e9e7b9587486b122e6aae. 2024-11-24T22:14:30,602 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bd6592c37bb2:0-1 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732486460633.63842f73863e9e7b9587486b122e6aae. as already flushing 2024-11-24T22:14:30,602 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bd6592c37bb2:0-1 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732486460633.63842f73863e9e7b9587486b122e6aae. 2024-11-24T22:14:30,602 ERROR [RS_FLUSH_OPERATIONS-regionserver/bd6592c37bb2:0-1 {event_type=RS_FLUSH_REGIONS, pid=43}] handler.RSProcedureHandler(58): pid=43 java.io.IOException: Unable to complete flush {ENCODED => 63842f73863e9e7b9587486b122e6aae, NAME => 'TestAcidGuarantees,,1732486460633.63842f73863e9e7b9587486b122e6aae.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T22:14:30,603 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bd6592c37bb2:0-1 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=43 java.io.IOException: Unable to complete flush {ENCODED => 63842f73863e9e7b9587486b122e6aae, NAME => 'TestAcidGuarantees,,1732486460633.63842f73863e9e7b9587486b122e6aae.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T22:14:30,604 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37701 {}] master.HMaster(4177): Remote procedure failed, pid=43 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 63842f73863e9e7b9587486b122e6aae, NAME => 'TestAcidGuarantees,,1732486460633.63842f73863e9e7b9587486b122e6aae.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 63842f73863e9e7b9587486b122e6aae, NAME => 'TestAcidGuarantees,,1732486460633.63842f73863e9e7b9587486b122e6aae.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T22:14:30,752 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40087 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=63842f73863e9e7b9587486b122e6aae, server=bd6592c37bb2,40087,1732486423713 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-24T22:14:30,752 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40087 {}] ipc.CallRunner(138): callId: 47 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40202 deadline: 1732486530750, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=63842f73863e9e7b9587486b122e6aae, server=bd6592c37bb2,40087,1732486423713 2024-11-24T22:14:30,752 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732486460633.63842f73863e9e7b9587486b122e6aae., hostname=bd6592c37bb2,40087,1732486423713, seqNum=5 , the old value is region=TestAcidGuarantees,,1732486460633.63842f73863e9e7b9587486b122e6aae., hostname=bd6592c37bb2,40087,1732486423713, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=63842f73863e9e7b9587486b122e6aae, server=bd6592c37bb2,40087,1732486423713 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T22:14:30,752 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732486460633.63842f73863e9e7b9587486b122e6aae., hostname=bd6592c37bb2,40087,1732486423713, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=63842f73863e9e7b9587486b122e6aae, server=bd6592c37bb2,40087,1732486423713 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T22:14:30,752 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732486460633.63842f73863e9e7b9587486b122e6aae., hostname=bd6592c37bb2,40087,1732486423713, seqNum=5 because the exception is null or not the one we care about 2024-11-24T22:14:30,757 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=40087 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=43 2024-11-24T22:14:30,757 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bd6592c37bb2:0-2 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732486460633.63842f73863e9e7b9587486b122e6aae. 2024-11-24T22:14:30,757 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bd6592c37bb2:0-2 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732486460633.63842f73863e9e7b9587486b122e6aae. as already flushing 2024-11-24T22:14:30,757 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bd6592c37bb2:0-2 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732486460633.63842f73863e9e7b9587486b122e6aae. 2024-11-24T22:14:30,757 ERROR [RS_FLUSH_OPERATIONS-regionserver/bd6592c37bb2:0-2 {event_type=RS_FLUSH_REGIONS, pid=43}] handler.RSProcedureHandler(58): pid=43 java.io.IOException: Unable to complete flush {ENCODED => 63842f73863e9e7b9587486b122e6aae, NAME => 'TestAcidGuarantees,,1732486460633.63842f73863e9e7b9587486b122e6aae.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T22:14:30,757 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bd6592c37bb2:0-2 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=43 java.io.IOException: Unable to complete flush {ENCODED => 63842f73863e9e7b9587486b122e6aae, NAME => 'TestAcidGuarantees,,1732486460633.63842f73863e9e7b9587486b122e6aae.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T22:14:30,759 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37701 {}] master.HMaster(4177): Remote procedure failed, pid=43 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 63842f73863e9e7b9587486b122e6aae, NAME => 'TestAcidGuarantees,,1732486460633.63842f73863e9e7b9587486b122e6aae.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 63842f73863e9e7b9587486b122e6aae, NAME => 'TestAcidGuarantees,,1732486460633.63842f73863e9e7b9587486b122e6aae.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T22:14:30,761 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37701 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=42 2024-11-24T22:14:30,889 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=158, memsize=40.3 K, hasBloomFilter=true, into tmp file hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/63842f73863e9e7b9587486b122e6aae/.tmp/A/a9d13b63e2ed4a508c48dec40823e1c0 2024-11-24T22:14:30,912 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=40087 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=43 2024-11-24T22:14:30,912 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bd6592c37bb2:0-0 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732486460633.63842f73863e9e7b9587486b122e6aae. 2024-11-24T22:14:30,912 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bd6592c37bb2:0-0 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732486460633.63842f73863e9e7b9587486b122e6aae. as already flushing 2024-11-24T22:14:30,912 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bd6592c37bb2:0-0 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732486460633.63842f73863e9e7b9587486b122e6aae. 2024-11-24T22:14:30,912 ERROR [RS_FLUSH_OPERATIONS-regionserver/bd6592c37bb2:0-0 {event_type=RS_FLUSH_REGIONS, pid=43}] handler.RSProcedureHandler(58): pid=43 java.io.IOException: Unable to complete flush {ENCODED => 63842f73863e9e7b9587486b122e6aae, NAME => 'TestAcidGuarantees,,1732486460633.63842f73863e9e7b9587486b122e6aae.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T22:14:30,912 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bd6592c37bb2:0-0 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=43 java.io.IOException: Unable to complete flush {ENCODED => 63842f73863e9e7b9587486b122e6aae, NAME => 'TestAcidGuarantees,,1732486460633.63842f73863e9e7b9587486b122e6aae.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T22:14:30,913 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37701 {}] master.HMaster(4177): Remote procedure failed, pid=43 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 63842f73863e9e7b9587486b122e6aae, NAME => 'TestAcidGuarantees,,1732486460633.63842f73863e9e7b9587486b122e6aae.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 63842f73863e9e7b9587486b122e6aae, NAME => 'TestAcidGuarantees,,1732486460633.63842f73863e9e7b9587486b122e6aae.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T22:14:30,919 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/63842f73863e9e7b9587486b122e6aae/.tmp/B/6b1102e285ca494e9044ea4767549dc0 is 50, key is test_row_0/B:col10/1732486470044/Put/seqid=0 2024-11-24T22:14:30,932 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45929 is added to blk_1073741977_1153 (size=12151) 2024-11-24T22:14:30,933 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=40.25 KB at sequenceid=158 (bloomFilter=true), to=hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/63842f73863e9e7b9587486b122e6aae/.tmp/B/6b1102e285ca494e9044ea4767549dc0 2024-11-24T22:14:30,943 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/63842f73863e9e7b9587486b122e6aae/.tmp/C/222b20f7e0474cf6b7bdaea5d17b20e2 is 50, key is test_row_0/C:col10/1732486470044/Put/seqid=0 2024-11-24T22:14:30,951 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45929 is added to blk_1073741978_1154 (size=12151) 2024-11-24T22:14:30,951 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=8.95 KB at sequenceid=158 (bloomFilter=true), to=hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/63842f73863e9e7b9587486b122e6aae/.tmp/C/222b20f7e0474cf6b7bdaea5d17b20e2 2024-11-24T22:14:30,956 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/63842f73863e9e7b9587486b122e6aae/.tmp/A/a9d13b63e2ed4a508c48dec40823e1c0 as hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/63842f73863e9e7b9587486b122e6aae/A/a9d13b63e2ed4a508c48dec40823e1c0 2024-11-24T22:14:30,960 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/63842f73863e9e7b9587486b122e6aae/A/a9d13b63e2ed4a508c48dec40823e1c0, entries=200, sequenceid=158, filesize=38.8 K 2024-11-24T22:14:30,961 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/63842f73863e9e7b9587486b122e6aae/.tmp/B/6b1102e285ca494e9044ea4767549dc0 as hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/63842f73863e9e7b9587486b122e6aae/B/6b1102e285ca494e9044ea4767549dc0 2024-11-24T22:14:30,965 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/63842f73863e9e7b9587486b122e6aae/B/6b1102e285ca494e9044ea4767549dc0, entries=150, sequenceid=158, filesize=11.9 K 2024-11-24T22:14:30,966 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/63842f73863e9e7b9587486b122e6aae/.tmp/C/222b20f7e0474cf6b7bdaea5d17b20e2 as hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/63842f73863e9e7b9587486b122e6aae/C/222b20f7e0474cf6b7bdaea5d17b20e2 2024-11-24T22:14:30,969 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/63842f73863e9e7b9587486b122e6aae/C/222b20f7e0474cf6b7bdaea5d17b20e2, entries=150, sequenceid=158, filesize=11.9 K 2024-11-24T22:14:30,970 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~89.45 KB/91600, heapSize ~262.06 KB/268352, currentSize=87.22 KB/89310 for 63842f73863e9e7b9587486b122e6aae in 924ms, sequenceid=158, compaction requested=true 2024-11-24T22:14:30,970 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 63842f73863e9e7b9587486b122e6aae: 2024-11-24T22:14:30,970 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 63842f73863e9e7b9587486b122e6aae:A, priority=-2147483648, current under compaction store size is 1 2024-11-24T22:14:30,971 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-24T22:14:30,971 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 63842f73863e9e7b9587486b122e6aae:B, priority=-2147483648, current under compaction store size is 2 2024-11-24T22:14:30,971 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-24T22:14:30,971 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 63842f73863e9e7b9587486b122e6aae:C, priority=-2147483648, current under compaction store size is 3 2024-11-24T22:14:30,971 DEBUG [RS:0;bd6592c37bb2:40087-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-24T22:14:30,971 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-24T22:14:30,971 DEBUG [RS:0;bd6592c37bb2:40087-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-24T22:14:30,972 DEBUG [RS:0;bd6592c37bb2:40087-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 132970 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-24T22:14:30,972 DEBUG [RS:0;bd6592c37bb2:40087-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 48510 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-24T22:14:30,973 DEBUG [RS:0;bd6592c37bb2:40087-shortCompactions-0 {}] regionserver.HStore(1541): 63842f73863e9e7b9587486b122e6aae/A is initiating minor compaction (all files) 2024-11-24T22:14:30,973 DEBUG [RS:0;bd6592c37bb2:40087-longCompactions-0 {}] regionserver.HStore(1541): 63842f73863e9e7b9587486b122e6aae/B is initiating minor compaction (all files) 2024-11-24T22:14:30,973 INFO [RS:0;bd6592c37bb2:40087-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 63842f73863e9e7b9587486b122e6aae/A in TestAcidGuarantees,,1732486460633.63842f73863e9e7b9587486b122e6aae. 2024-11-24T22:14:30,973 INFO [RS:0;bd6592c37bb2:40087-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/63842f73863e9e7b9587486b122e6aae/A/7a8e21a07dbe4b9f9a240d4f840c9c50, hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/63842f73863e9e7b9587486b122e6aae/A/79994ec69119412bacc781ba6aa897bb, hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/63842f73863e9e7b9587486b122e6aae/A/35f72322d95641559d1e43da1cc0a431, hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/63842f73863e9e7b9587486b122e6aae/A/a9d13b63e2ed4a508c48dec40823e1c0] into tmpdir=hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/63842f73863e9e7b9587486b122e6aae/.tmp, totalSize=129.9 K 2024-11-24T22:14:30,973 INFO [RS:0;bd6592c37bb2:40087-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 63842f73863e9e7b9587486b122e6aae/B in TestAcidGuarantees,,1732486460633.63842f73863e9e7b9587486b122e6aae. 2024-11-24T22:14:30,973 INFO [RS:0;bd6592c37bb2:40087-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=12 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1732486460633.63842f73863e9e7b9587486b122e6aae. 2024-11-24T22:14:30,973 DEBUG [RS:0;bd6592c37bb2:40087-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1732486460633.63842f73863e9e7b9587486b122e6aae. files: [hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/63842f73863e9e7b9587486b122e6aae/A/7a8e21a07dbe4b9f9a240d4f840c9c50, hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/63842f73863e9e7b9587486b122e6aae/A/79994ec69119412bacc781ba6aa897bb, hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/63842f73863e9e7b9587486b122e6aae/A/35f72322d95641559d1e43da1cc0a431, hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/63842f73863e9e7b9587486b122e6aae/A/a9d13b63e2ed4a508c48dec40823e1c0] 2024-11-24T22:14:30,973 INFO [RS:0;bd6592c37bb2:40087-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/63842f73863e9e7b9587486b122e6aae/B/85d047d5e1164e14a0ecf07a70e8146c, hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/63842f73863e9e7b9587486b122e6aae/B/5eda402104654194b9e33e227dc03d50, hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/63842f73863e9e7b9587486b122e6aae/B/7b7482699d6743d1874a0762ec11ae0e, hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/63842f73863e9e7b9587486b122e6aae/B/6b1102e285ca494e9044ea4767549dc0] into tmpdir=hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/63842f73863e9e7b9587486b122e6aae/.tmp, totalSize=47.4 K 2024-11-24T22:14:30,973 DEBUG [RS:0;bd6592c37bb2:40087-shortCompactions-0 {}] compactions.Compactor(225): Compacting 7a8e21a07dbe4b9f9a240d4f840c9c50, keycount=150, bloomtype=ROW, size=30.4 K, encoding=NONE, compression=NONE, seqNum=102, earliestPutTs=1732486466601 2024-11-24T22:14:30,973 DEBUG [RS:0;bd6592c37bb2:40087-longCompactions-0 {}] compactions.Compactor(225): Compacting 85d047d5e1164e14a0ecf07a70e8146c, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=102, earliestPutTs=1732486466601 2024-11-24T22:14:30,974 DEBUG [RS:0;bd6592c37bb2:40087-longCompactions-0 {}] compactions.Compactor(225): Compacting 5eda402104654194b9e33e227dc03d50, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=122, earliestPutTs=1732486467804 2024-11-24T22:14:30,974 DEBUG [RS:0;bd6592c37bb2:40087-shortCompactions-0 {}] compactions.Compactor(225): Compacting 79994ec69119412bacc781ba6aa897bb, keycount=150, bloomtype=ROW, size=30.2 K, encoding=NONE, compression=NONE, seqNum=122, earliestPutTs=1732486467804 2024-11-24T22:14:30,974 DEBUG [RS:0;bd6592c37bb2:40087-longCompactions-0 {}] compactions.Compactor(225): Compacting 7b7482699d6743d1874a0762ec11ae0e, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=137, earliestPutTs=1732486468082 2024-11-24T22:14:30,974 DEBUG [RS:0;bd6592c37bb2:40087-shortCompactions-0 {}] compactions.Compactor(225): Compacting 35f72322d95641559d1e43da1cc0a431, keycount=150, bloomtype=ROW, size=30.4 K, encoding=NONE, compression=NONE, seqNum=137, earliestPutTs=1732486468082 2024-11-24T22:14:30,977 DEBUG [RS:0;bd6592c37bb2:40087-longCompactions-0 {}] compactions.Compactor(225): Compacting 6b1102e285ca494e9044ea4767549dc0, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=158, earliestPutTs=1732486469042 2024-11-24T22:14:30,977 DEBUG [RS:0;bd6592c37bb2:40087-shortCompactions-0 {}] compactions.Compactor(225): Compacting a9d13b63e2ed4a508c48dec40823e1c0, keycount=200, bloomtype=ROW, size=38.8 K, encoding=NONE, compression=NONE, seqNum=158, earliestPutTs=1732486469034 2024-11-24T22:14:31,003 INFO [RS:0;bd6592c37bb2:40087-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 63842f73863e9e7b9587486b122e6aae#B#compaction#129 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-24T22:14:31,004 DEBUG [RS:0;bd6592c37bb2:40087-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/63842f73863e9e7b9587486b122e6aae/.tmp/B/ed25299c01a145d88eaa3f03fd0b147e is 50, key is test_row_0/B:col10/1732486470044/Put/seqid=0 2024-11-24T22:14:31,007 INFO [RS:0;bd6592c37bb2:40087-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=63842f73863e9e7b9587486b122e6aae] 2024-11-24T22:14:31,013 DEBUG [RS:0;bd6592c37bb2:40087-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e2024112451d9ff68dcbc4c189c4fa73c8cbcd3b9_63842f73863e9e7b9587486b122e6aae store=[table=TestAcidGuarantees family=A region=63842f73863e9e7b9587486b122e6aae] 2024-11-24T22:14:31,017 DEBUG [RS:0;bd6592c37bb2:40087-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e2024112451d9ff68dcbc4c189c4fa73c8cbcd3b9_63842f73863e9e7b9587486b122e6aae, store=[table=TestAcidGuarantees family=A region=63842f73863e9e7b9587486b122e6aae] 2024-11-24T22:14:31,017 DEBUG [RS:0;bd6592c37bb2:40087-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024112451d9ff68dcbc4c189c4fa73c8cbcd3b9_63842f73863e9e7b9587486b122e6aae because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=63842f73863e9e7b9587486b122e6aae] 2024-11-24T22:14:31,063 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45929 is added to blk_1073741979_1155 (size=12493) 2024-11-24T22:14:31,065 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=40087 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=43 2024-11-24T22:14:31,066 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bd6592c37bb2:0-1 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732486460633.63842f73863e9e7b9587486b122e6aae. 2024-11-24T22:14:31,066 INFO [RS_FLUSH_OPERATIONS-regionserver/bd6592c37bb2:0-1 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.HRegion(2902): Flushing 63842f73863e9e7b9587486b122e6aae 3/3 column families, dataSize=87.22 KB heapSize=259.73 KB 2024-11-24T22:14:31,066 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bd6592c37bb2:0-1 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 63842f73863e9e7b9587486b122e6aae, store=A 2024-11-24T22:14:31,067 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bd6592c37bb2:0-1 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-24T22:14:31,067 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bd6592c37bb2:0-1 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 63842f73863e9e7b9587486b122e6aae, store=B 2024-11-24T22:14:31,067 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bd6592c37bb2:0-1 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-24T22:14:31,067 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bd6592c37bb2:0-1 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 63842f73863e9e7b9587486b122e6aae, store=C 2024-11-24T22:14:31,067 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bd6592c37bb2:0-1 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-24T22:14:31,076 DEBUG [RS:0;bd6592c37bb2:40087-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/63842f73863e9e7b9587486b122e6aae/.tmp/B/ed25299c01a145d88eaa3f03fd0b147e as hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/63842f73863e9e7b9587486b122e6aae/B/ed25299c01a145d88eaa3f03fd0b147e 2024-11-24T22:14:31,090 INFO [RS:0;bd6592c37bb2:40087-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 63842f73863e9e7b9587486b122e6aae/B of 63842f73863e9e7b9587486b122e6aae into ed25299c01a145d88eaa3f03fd0b147e(size=12.2 K), total size for store is 12.2 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-24T22:14:31,090 DEBUG [RS:0;bd6592c37bb2:40087-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 63842f73863e9e7b9587486b122e6aae: 2024-11-24T22:14:31,090 INFO [RS:0;bd6592c37bb2:40087-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732486460633.63842f73863e9e7b9587486b122e6aae., storeName=63842f73863e9e7b9587486b122e6aae/B, priority=12, startTime=1732486470971; duration=0sec 2024-11-24T22:14:31,090 DEBUG [RS:0;bd6592c37bb2:40087-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-24T22:14:31,090 DEBUG [RS:0;bd6592c37bb2:40087-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 63842f73863e9e7b9587486b122e6aae:B 2024-11-24T22:14:31,090 DEBUG [RS:0;bd6592c37bb2:40087-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-24T22:14:31,094 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bd6592c37bb2:0-1 {event_type=RS_FLUSH_REGIONS, pid=43}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411243ad647717a8d43cda58954ccaded35ca_63842f73863e9e7b9587486b122e6aae is 50, key is test_row_0/A:col10/1732486470084/Put/seqid=0 2024-11-24T22:14:31,094 DEBUG [RS:0;bd6592c37bb2:40087-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 48510 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-24T22:14:31,094 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45929 is added to blk_1073741980_1156 (size=4469) 2024-11-24T22:14:31,094 DEBUG [RS:0;bd6592c37bb2:40087-longCompactions-0 {}] regionserver.HStore(1541): 63842f73863e9e7b9587486b122e6aae/C is initiating minor compaction (all files) 2024-11-24T22:14:31,094 INFO [RS:0;bd6592c37bb2:40087-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 63842f73863e9e7b9587486b122e6aae/C in TestAcidGuarantees,,1732486460633.63842f73863e9e7b9587486b122e6aae. 2024-11-24T22:14:31,094 INFO [RS:0;bd6592c37bb2:40087-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/63842f73863e9e7b9587486b122e6aae/C/827f6af5e50f48edb4d996590552ae1d, hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/63842f73863e9e7b9587486b122e6aae/C/6942817daf0640c5b53d955de9391e7c, hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/63842f73863e9e7b9587486b122e6aae/C/4d52a0c91d794d46956f4bd1629386f4, hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/63842f73863e9e7b9587486b122e6aae/C/222b20f7e0474cf6b7bdaea5d17b20e2] into tmpdir=hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/63842f73863e9e7b9587486b122e6aae/.tmp, totalSize=47.4 K 2024-11-24T22:14:31,095 DEBUG [RS:0;bd6592c37bb2:40087-longCompactions-0 {}] compactions.Compactor(225): Compacting 827f6af5e50f48edb4d996590552ae1d, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=102, earliestPutTs=1732486466601 2024-11-24T22:14:31,095 INFO [RS:0;bd6592c37bb2:40087-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 63842f73863e9e7b9587486b122e6aae#A#compaction#130 average throughput is 0.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-24T22:14:31,096 DEBUG [RS:0;bd6592c37bb2:40087-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/63842f73863e9e7b9587486b122e6aae/.tmp/A/a4e17bb33ebb463ea0574c07bb661b48 is 175, key is test_row_0/A:col10/1732486470044/Put/seqid=0 2024-11-24T22:14:31,096 DEBUG [RS:0;bd6592c37bb2:40087-longCompactions-0 {}] compactions.Compactor(225): Compacting 6942817daf0640c5b53d955de9391e7c, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=122, earliestPutTs=1732486467804 2024-11-24T22:14:31,097 DEBUG [RS:0;bd6592c37bb2:40087-longCompactions-0 {}] compactions.Compactor(225): Compacting 4d52a0c91d794d46956f4bd1629386f4, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=137, earliestPutTs=1732486468082 2024-11-24T22:14:31,097 DEBUG [RS:0;bd6592c37bb2:40087-longCompactions-0 {}] compactions.Compactor(225): Compacting 222b20f7e0474cf6b7bdaea5d17b20e2, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=158, earliestPutTs=1732486469042 2024-11-24T22:14:31,142 INFO [RS:0;bd6592c37bb2:40087-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 63842f73863e9e7b9587486b122e6aae#C#compaction#132 average throughput is 2.18 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-24T22:14:31,143 DEBUG [RS:0;bd6592c37bb2:40087-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/63842f73863e9e7b9587486b122e6aae/.tmp/C/89494cbdc6fb4d5da656158a799b399e is 50, key is test_row_0/C:col10/1732486470044/Put/seqid=0 2024-11-24T22:14:31,147 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45929 is added to blk_1073741982_1158 (size=31447) 2024-11-24T22:14:31,154 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45929 is added to blk_1073741981_1157 (size=12304) 2024-11-24T22:14:31,178 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45929 is added to blk_1073741983_1159 (size=12493) 2024-11-24T22:14:31,185 DEBUG [RS:0;bd6592c37bb2:40087-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/63842f73863e9e7b9587486b122e6aae/.tmp/C/89494cbdc6fb4d5da656158a799b399e as hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/63842f73863e9e7b9587486b122e6aae/C/89494cbdc6fb4d5da656158a799b399e 2024-11-24T22:14:31,195 INFO [RS:0;bd6592c37bb2:40087-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 63842f73863e9e7b9587486b122e6aae/C of 63842f73863e9e7b9587486b122e6aae into 89494cbdc6fb4d5da656158a799b399e(size=12.2 K), total size for store is 12.2 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-24T22:14:31,195 DEBUG [RS:0;bd6592c37bb2:40087-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 63842f73863e9e7b9587486b122e6aae: 2024-11-24T22:14:31,195 INFO [RS:0;bd6592c37bb2:40087-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732486460633.63842f73863e9e7b9587486b122e6aae., storeName=63842f73863e9e7b9587486b122e6aae/C, priority=12, startTime=1732486470971; duration=0sec 2024-11-24T22:14:31,195 DEBUG [RS:0;bd6592c37bb2:40087-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-24T22:14:31,195 DEBUG [RS:0;bd6592c37bb2:40087-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 63842f73863e9e7b9587486b122e6aae:C 2024-11-24T22:14:31,217 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40087 {}] regionserver.HRegion(8855): Flush requested on 63842f73863e9e7b9587486b122e6aae 2024-11-24T22:14:31,217 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732486460633.63842f73863e9e7b9587486b122e6aae. as already flushing 2024-11-24T22:14:31,257 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40087 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=63842f73863e9e7b9587486b122e6aae, server=bd6592c37bb2,40087,1732486423713 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-24T22:14:31,257 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40087 {}] ipc.CallRunner(138): callId: 62 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40244 deadline: 1732486531253, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=63842f73863e9e7b9587486b122e6aae, server=bd6592c37bb2,40087,1732486423713 2024-11-24T22:14:31,257 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40087 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=63842f73863e9e7b9587486b122e6aae, server=bd6592c37bb2,40087,1732486423713 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-24T22:14:31,258 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40087 {}] ipc.CallRunner(138): callId: 70 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40206 deadline: 1732486531254, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=63842f73863e9e7b9587486b122e6aae, server=bd6592c37bb2,40087,1732486423713 2024-11-24T22:14:31,258 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732486460633.63842f73863e9e7b9587486b122e6aae., hostname=bd6592c37bb2,40087,1732486423713, seqNum=5 , the old value is region=TestAcidGuarantees,,1732486460633.63842f73863e9e7b9587486b122e6aae., hostname=bd6592c37bb2,40087,1732486423713, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=63842f73863e9e7b9587486b122e6aae, server=bd6592c37bb2,40087,1732486423713 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T22:14:31,258 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732486460633.63842f73863e9e7b9587486b122e6aae., hostname=bd6592c37bb2,40087,1732486423713, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=63842f73863e9e7b9587486b122e6aae, server=bd6592c37bb2,40087,1732486423713 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T22:14:31,258 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732486460633.63842f73863e9e7b9587486b122e6aae., hostname=bd6592c37bb2,40087,1732486423713, seqNum=5 because the exception is null or not the one we care about 2024-11-24T22:14:31,259 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732486460633.63842f73863e9e7b9587486b122e6aae., hostname=bd6592c37bb2,40087,1732486423713, seqNum=5 , the old value is region=TestAcidGuarantees,,1732486460633.63842f73863e9e7b9587486b122e6aae., hostname=bd6592c37bb2,40087,1732486423713, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=63842f73863e9e7b9587486b122e6aae, server=bd6592c37bb2,40087,1732486423713 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T22:14:31,259 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732486460633.63842f73863e9e7b9587486b122e6aae., hostname=bd6592c37bb2,40087,1732486423713, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=63842f73863e9e7b9587486b122e6aae, server=bd6592c37bb2,40087,1732486423713 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T22:14:31,259 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732486460633.63842f73863e9e7b9587486b122e6aae., hostname=bd6592c37bb2,40087,1732486423713, seqNum=5 because the exception is null or not the one we care about 2024-11-24T22:14:31,265 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40087 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=63842f73863e9e7b9587486b122e6aae, server=bd6592c37bb2,40087,1732486423713 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-24T22:14:31,265 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40087 {}] ipc.CallRunner(138): callId: 48 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40202 deadline: 1732486531261, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=63842f73863e9e7b9587486b122e6aae, server=bd6592c37bb2,40087,1732486423713 2024-11-24T22:14:31,265 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732486460633.63842f73863e9e7b9587486b122e6aae., hostname=bd6592c37bb2,40087,1732486423713, seqNum=5 , the old value is region=TestAcidGuarantees,,1732486460633.63842f73863e9e7b9587486b122e6aae., hostname=bd6592c37bb2,40087,1732486423713, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=63842f73863e9e7b9587486b122e6aae, server=bd6592c37bb2,40087,1732486423713 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T22:14:31,265 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732486460633.63842f73863e9e7b9587486b122e6aae., hostname=bd6592c37bb2,40087,1732486423713, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=63842f73863e9e7b9587486b122e6aae, server=bd6592c37bb2,40087,1732486423713 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T22:14:31,266 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732486460633.63842f73863e9e7b9587486b122e6aae., hostname=bd6592c37bb2,40087,1732486423713, seqNum=5 because the exception is null or not the one we care about 2024-11-24T22:14:31,271 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37701 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=42 2024-11-24T22:14:31,364 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40087 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=63842f73863e9e7b9587486b122e6aae, server=bd6592c37bb2,40087,1732486423713 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-24T22:14:31,364 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40087 {}] ipc.CallRunner(138): callId: 71 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40206 deadline: 1732486531360, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=63842f73863e9e7b9587486b122e6aae, server=bd6592c37bb2,40087,1732486423713 2024-11-24T22:14:31,364 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732486460633.63842f73863e9e7b9587486b122e6aae., hostname=bd6592c37bb2,40087,1732486423713, seqNum=5 , the old value is region=TestAcidGuarantees,,1732486460633.63842f73863e9e7b9587486b122e6aae., hostname=bd6592c37bb2,40087,1732486423713, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=63842f73863e9e7b9587486b122e6aae, server=bd6592c37bb2,40087,1732486423713 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T22:14:31,364 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732486460633.63842f73863e9e7b9587486b122e6aae., hostname=bd6592c37bb2,40087,1732486423713, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=63842f73863e9e7b9587486b122e6aae, server=bd6592c37bb2,40087,1732486423713 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T22:14:31,364 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732486460633.63842f73863e9e7b9587486b122e6aae., hostname=bd6592c37bb2,40087,1732486423713, seqNum=5 because the exception is null or not the one we care about 2024-11-24T22:14:31,371 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40087 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=63842f73863e9e7b9587486b122e6aae, server=bd6592c37bb2,40087,1732486423713 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-24T22:14:31,372 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40087 {}] ipc.CallRunner(138): callId: 63 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40244 deadline: 1732486531371, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=63842f73863e9e7b9587486b122e6aae, server=bd6592c37bb2,40087,1732486423713 2024-11-24T22:14:31,373 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732486460633.63842f73863e9e7b9587486b122e6aae., hostname=bd6592c37bb2,40087,1732486423713, seqNum=5 , the old value is region=TestAcidGuarantees,,1732486460633.63842f73863e9e7b9587486b122e6aae., hostname=bd6592c37bb2,40087,1732486423713, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=63842f73863e9e7b9587486b122e6aae, server=bd6592c37bb2,40087,1732486423713 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T22:14:31,373 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732486460633.63842f73863e9e7b9587486b122e6aae., hostname=bd6592c37bb2,40087,1732486423713, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=63842f73863e9e7b9587486b122e6aae, server=bd6592c37bb2,40087,1732486423713 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T22:14:31,373 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732486460633.63842f73863e9e7b9587486b122e6aae., hostname=bd6592c37bb2,40087,1732486423713, seqNum=5 because the exception is null or not the one we care about 2024-11-24T22:14:31,553 DEBUG [RS:0;bd6592c37bb2:40087-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/63842f73863e9e7b9587486b122e6aae/.tmp/A/a4e17bb33ebb463ea0574c07bb661b48 as hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/63842f73863e9e7b9587486b122e6aae/A/a4e17bb33ebb463ea0574c07bb661b48 2024-11-24T22:14:31,554 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bd6592c37bb2:0-1 {event_type=RS_FLUSH_REGIONS, pid=43}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T22:14:31,559 INFO [RS:0;bd6592c37bb2:40087-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 63842f73863e9e7b9587486b122e6aae/A of 63842f73863e9e7b9587486b122e6aae into a4e17bb33ebb463ea0574c07bb661b48(size=30.7 K), total size for store is 30.7 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-24T22:14:31,559 DEBUG [RS:0;bd6592c37bb2:40087-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 63842f73863e9e7b9587486b122e6aae: 2024-11-24T22:14:31,559 INFO [RS:0;bd6592c37bb2:40087-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732486460633.63842f73863e9e7b9587486b122e6aae., storeName=63842f73863e9e7b9587486b122e6aae/A, priority=12, startTime=1732486470970; duration=0sec 2024-11-24T22:14:31,559 DEBUG [RS:0;bd6592c37bb2:40087-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-24T22:14:31,559 DEBUG [RS:0;bd6592c37bb2:40087-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 63842f73863e9e7b9587486b122e6aae:A 2024-11-24T22:14:31,559 INFO [RS_FLUSH_OPERATIONS-regionserver/bd6592c37bb2:0-1 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411243ad647717a8d43cda58954ccaded35ca_63842f73863e9e7b9587486b122e6aae to hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411243ad647717a8d43cda58954ccaded35ca_63842f73863e9e7b9587486b122e6aae 2024-11-24T22:14:31,560 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bd6592c37bb2:0-1 {event_type=RS_FLUSH_REGIONS, pid=43}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/63842f73863e9e7b9587486b122e6aae/.tmp/A/54b3fdfd1b984f089b1a2242f0d9d9b2, store: [table=TestAcidGuarantees family=A region=63842f73863e9e7b9587486b122e6aae] 2024-11-24T22:14:31,561 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bd6592c37bb2:0-1 {event_type=RS_FLUSH_REGIONS, pid=43}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/63842f73863e9e7b9587486b122e6aae/.tmp/A/54b3fdfd1b984f089b1a2242f0d9d9b2 is 175, key is test_row_0/A:col10/1732486470084/Put/seqid=0 2024-11-24T22:14:31,564 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45929 is added to blk_1073741984_1160 (size=31105) 2024-11-24T22:14:31,565 INFO [RS_FLUSH_OPERATIONS-regionserver/bd6592c37bb2:0-1 {event_type=RS_FLUSH_REGIONS, pid=43}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=174, memsize=29.1 K, hasBloomFilter=true, into tmp file hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/63842f73863e9e7b9587486b122e6aae/.tmp/A/54b3fdfd1b984f089b1a2242f0d9d9b2 2024-11-24T22:14:31,573 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bd6592c37bb2:0-1 {event_type=RS_FLUSH_REGIONS, pid=43}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/63842f73863e9e7b9587486b122e6aae/.tmp/B/8546de4e512a4074a0c26e2f30373177 is 50, key is test_row_0/B:col10/1732486470084/Put/seqid=0 2024-11-24T22:14:31,573 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40087 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=63842f73863e9e7b9587486b122e6aae, server=bd6592c37bb2,40087,1732486423713 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-24T22:14:31,573 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40087 {}] ipc.CallRunner(138): callId: 72 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40206 deadline: 1732486531570, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=63842f73863e9e7b9587486b122e6aae, server=bd6592c37bb2,40087,1732486423713 2024-11-24T22:14:31,574 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732486460633.63842f73863e9e7b9587486b122e6aae., hostname=bd6592c37bb2,40087,1732486423713, seqNum=5 , the old value is region=TestAcidGuarantees,,1732486460633.63842f73863e9e7b9587486b122e6aae., hostname=bd6592c37bb2,40087,1732486423713, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=63842f73863e9e7b9587486b122e6aae, server=bd6592c37bb2,40087,1732486423713 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T22:14:31,574 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732486460633.63842f73863e9e7b9587486b122e6aae., hostname=bd6592c37bb2,40087,1732486423713, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=63842f73863e9e7b9587486b122e6aae, server=bd6592c37bb2,40087,1732486423713 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T22:14:31,574 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732486460633.63842f73863e9e7b9587486b122e6aae., hostname=bd6592c37bb2,40087,1732486423713, seqNum=5 because the exception is null or not the one we care about 2024-11-24T22:14:31,577 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45929 is added to blk_1073741985_1161 (size=12151) 2024-11-24T22:14:31,585 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40087 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=63842f73863e9e7b9587486b122e6aae, server=bd6592c37bb2,40087,1732486423713 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-24T22:14:31,585 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40087 {}] ipc.CallRunner(138): callId: 64 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40244 deadline: 1732486531581, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=63842f73863e9e7b9587486b122e6aae, server=bd6592c37bb2,40087,1732486423713 2024-11-24T22:14:31,585 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732486460633.63842f73863e9e7b9587486b122e6aae., hostname=bd6592c37bb2,40087,1732486423713, seqNum=5 , the old value is region=TestAcidGuarantees,,1732486460633.63842f73863e9e7b9587486b122e6aae., hostname=bd6592c37bb2,40087,1732486423713, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=63842f73863e9e7b9587486b122e6aae, server=bd6592c37bb2,40087,1732486423713 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T22:14:31,585 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732486460633.63842f73863e9e7b9587486b122e6aae., hostname=bd6592c37bb2,40087,1732486423713, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=63842f73863e9e7b9587486b122e6aae, server=bd6592c37bb2,40087,1732486423713 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T22:14:31,585 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732486460633.63842f73863e9e7b9587486b122e6aae., hostname=bd6592c37bb2,40087,1732486423713, seqNum=5 because the exception is null or not the one we care about 2024-11-24T22:14:31,882 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40087 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=63842f73863e9e7b9587486b122e6aae, server=bd6592c37bb2,40087,1732486423713 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-24T22:14:31,882 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40087 {}] ipc.CallRunner(138): callId: 73 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40206 deadline: 1732486531881, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=63842f73863e9e7b9587486b122e6aae, server=bd6592c37bb2,40087,1732486423713 2024-11-24T22:14:31,882 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732486460633.63842f73863e9e7b9587486b122e6aae., hostname=bd6592c37bb2,40087,1732486423713, seqNum=5 , the old value is region=TestAcidGuarantees,,1732486460633.63842f73863e9e7b9587486b122e6aae., hostname=bd6592c37bb2,40087,1732486423713, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=63842f73863e9e7b9587486b122e6aae, server=bd6592c37bb2,40087,1732486423713 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T22:14:31,883 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732486460633.63842f73863e9e7b9587486b122e6aae., hostname=bd6592c37bb2,40087,1732486423713, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=63842f73863e9e7b9587486b122e6aae, server=bd6592c37bb2,40087,1732486423713 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T22:14:31,883 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732486460633.63842f73863e9e7b9587486b122e6aae., hostname=bd6592c37bb2,40087,1732486423713, seqNum=5 because the exception is null or not the one we care about 2024-11-24T22:14:31,897 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40087 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=63842f73863e9e7b9587486b122e6aae, server=bd6592c37bb2,40087,1732486423713 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-24T22:14:31,897 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40087 {}] ipc.CallRunner(138): callId: 65 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40244 deadline: 1732486531891, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=63842f73863e9e7b9587486b122e6aae, server=bd6592c37bb2,40087,1732486423713 2024-11-24T22:14:31,897 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732486460633.63842f73863e9e7b9587486b122e6aae., hostname=bd6592c37bb2,40087,1732486423713, seqNum=5 , the old value is region=TestAcidGuarantees,,1732486460633.63842f73863e9e7b9587486b122e6aae., hostname=bd6592c37bb2,40087,1732486423713, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=63842f73863e9e7b9587486b122e6aae, server=bd6592c37bb2,40087,1732486423713 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T22:14:31,897 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732486460633.63842f73863e9e7b9587486b122e6aae., hostname=bd6592c37bb2,40087,1732486423713, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=63842f73863e9e7b9587486b122e6aae, server=bd6592c37bb2,40087,1732486423713 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T22:14:31,897 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732486460633.63842f73863e9e7b9587486b122e6aae., hostname=bd6592c37bb2,40087,1732486423713, seqNum=5 because the exception is null or not the one we care about 2024-11-24T22:14:31,978 INFO [RS_FLUSH_OPERATIONS-regionserver/bd6592c37bb2:0-1 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=29.07 KB at sequenceid=174 (bloomFilter=true), to=hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/63842f73863e9e7b9587486b122e6aae/.tmp/B/8546de4e512a4074a0c26e2f30373177 2024-11-24T22:14:31,988 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bd6592c37bb2:0-1 {event_type=RS_FLUSH_REGIONS, pid=43}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/63842f73863e9e7b9587486b122e6aae/.tmp/C/5c1e349f365f4cfda43b21302f209dec is 50, key is test_row_0/C:col10/1732486470084/Put/seqid=0 2024-11-24T22:14:31,998 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45929 is added to blk_1073741986_1162 (size=12151) 2024-11-24T22:14:32,277 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40087 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=63842f73863e9e7b9587486b122e6aae, server=bd6592c37bb2,40087,1732486423713 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-24T22:14:32,278 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40087 {}] ipc.CallRunner(138): callId: 37 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40232 deadline: 1732486532271, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=63842f73863e9e7b9587486b122e6aae, server=bd6592c37bb2,40087,1732486423713 2024-11-24T22:14:32,278 WARN [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRpcRetryingCaller(177): Call to bd6592c37bb2,40087,1732486423713 for 'test_row_1' in 63842f73863e9e7b9587486b122e6aae of TestAcidGuarantees failed, tries = 7, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 4183 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=63842f73863e9e7b9587486b122e6aae, server=bd6592c37bb2,40087,1732486423713 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor44.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=63842f73863e9e7b9587486b122e6aae, server=bd6592c37bb2,40087,1732486423713 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-24T22:14:32,279 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732486460633.63842f73863e9e7b9587486b122e6aae., hostname=bd6592c37bb2,40087,1732486423713, seqNum=5 , the old value is region=TestAcidGuarantees,,1732486460633.63842f73863e9e7b9587486b122e6aae., hostname=bd6592c37bb2,40087,1732486423713, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=63842f73863e9e7b9587486b122e6aae, server=bd6592c37bb2,40087,1732486423713 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T22:14:32,279 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732486460633.63842f73863e9e7b9587486b122e6aae., hostname=bd6592c37bb2,40087,1732486423713, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=63842f73863e9e7b9587486b122e6aae, server=bd6592c37bb2,40087,1732486423713 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T22:14:32,279 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732486460633.63842f73863e9e7b9587486b122e6aae., hostname=bd6592c37bb2,40087,1732486423713, seqNum=5 because the exception is null or not the one we care about 2024-11-24T22:14:32,281 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40087 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=63842f73863e9e7b9587486b122e6aae, server=bd6592c37bb2,40087,1732486423713 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-24T22:14:32,282 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40087 {}] ipc.CallRunner(138): callId: 37 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40226 deadline: 1732486532281, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=63842f73863e9e7b9587486b122e6aae, server=bd6592c37bb2,40087,1732486423713 2024-11-24T22:14:32,282 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40087 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=63842f73863e9e7b9587486b122e6aae, server=bd6592c37bb2,40087,1732486423713 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-24T22:14:32,282 WARN [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRpcRetryingCaller(177): Call to bd6592c37bb2,40087,1732486423713 for 'test_row_2' in 63842f73863e9e7b9587486b122e6aae of TestAcidGuarantees failed, tries = 7, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 4194 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=63842f73863e9e7b9587486b122e6aae, server=bd6592c37bb2,40087,1732486423713 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor44.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=63842f73863e9e7b9587486b122e6aae, server=bd6592c37bb2,40087,1732486423713 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-24T22:14:32,282 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40087 {}] ipc.CallRunner(138): callId: 49 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40202 deadline: 1732486532281, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=63842f73863e9e7b9587486b122e6aae, server=bd6592c37bb2,40087,1732486423713 2024-11-24T22:14:32,283 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732486460633.63842f73863e9e7b9587486b122e6aae., hostname=bd6592c37bb2,40087,1732486423713, seqNum=5 , the old value is region=TestAcidGuarantees,,1732486460633.63842f73863e9e7b9587486b122e6aae., hostname=bd6592c37bb2,40087,1732486423713, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=63842f73863e9e7b9587486b122e6aae, server=bd6592c37bb2,40087,1732486423713 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T22:14:32,283 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732486460633.63842f73863e9e7b9587486b122e6aae., hostname=bd6592c37bb2,40087,1732486423713, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=63842f73863e9e7b9587486b122e6aae, server=bd6592c37bb2,40087,1732486423713 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T22:14:32,283 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732486460633.63842f73863e9e7b9587486b122e6aae., hostname=bd6592c37bb2,40087,1732486423713, seqNum=5 because the exception is null or not the one we care about 2024-11-24T22:14:32,283 WARN [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRpcRetryingCaller(177): Call to bd6592c37bb2,40087,1732486423713 for 'test_row_0' in 63842f73863e9e7b9587486b122e6aae of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2168 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=63842f73863e9e7b9587486b122e6aae, server=bd6592c37bb2,40087,1732486423713 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor44.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=63842f73863e9e7b9587486b122e6aae, server=bd6592c37bb2,40087,1732486423713 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-24T22:14:32,284 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732486460633.63842f73863e9e7b9587486b122e6aae., hostname=bd6592c37bb2,40087,1732486423713, seqNum=5 , the old value is region=TestAcidGuarantees,,1732486460633.63842f73863e9e7b9587486b122e6aae., hostname=bd6592c37bb2,40087,1732486423713, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=63842f73863e9e7b9587486b122e6aae, server=bd6592c37bb2,40087,1732486423713 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T22:14:32,284 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732486460633.63842f73863e9e7b9587486b122e6aae., hostname=bd6592c37bb2,40087,1732486423713, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=63842f73863e9e7b9587486b122e6aae, server=bd6592c37bb2,40087,1732486423713 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T22:14:32,284 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732486460633.63842f73863e9e7b9587486b122e6aae., hostname=bd6592c37bb2,40087,1732486423713, seqNum=5 because the exception is null or not the one we care about 2024-11-24T22:14:32,291 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37701 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=42 2024-11-24T22:14:32,397 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40087 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=63842f73863e9e7b9587486b122e6aae, server=bd6592c37bb2,40087,1732486423713 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-24T22:14:32,397 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40087 {}] ipc.CallRunner(138): callId: 74 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40206 deadline: 1732486532390, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=63842f73863e9e7b9587486b122e6aae, server=bd6592c37bb2,40087,1732486423713 2024-11-24T22:14:32,399 INFO [RS_FLUSH_OPERATIONS-regionserver/bd6592c37bb2:0-1 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=29.07 KB at sequenceid=174 (bloomFilter=true), to=hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/63842f73863e9e7b9587486b122e6aae/.tmp/C/5c1e349f365f4cfda43b21302f209dec 2024-11-24T22:14:32,404 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bd6592c37bb2:0-1 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/63842f73863e9e7b9587486b122e6aae/.tmp/A/54b3fdfd1b984f089b1a2242f0d9d9b2 as hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/63842f73863e9e7b9587486b122e6aae/A/54b3fdfd1b984f089b1a2242f0d9d9b2 2024-11-24T22:14:32,405 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732486460633.63842f73863e9e7b9587486b122e6aae., hostname=bd6592c37bb2,40087,1732486423713, seqNum=5 , the old value is region=TestAcidGuarantees,,1732486460633.63842f73863e9e7b9587486b122e6aae., hostname=bd6592c37bb2,40087,1732486423713, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=63842f73863e9e7b9587486b122e6aae, server=bd6592c37bb2,40087,1732486423713 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T22:14:32,405 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732486460633.63842f73863e9e7b9587486b122e6aae., hostname=bd6592c37bb2,40087,1732486423713, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=63842f73863e9e7b9587486b122e6aae, server=bd6592c37bb2,40087,1732486423713 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T22:14:32,405 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732486460633.63842f73863e9e7b9587486b122e6aae., hostname=bd6592c37bb2,40087,1732486423713, seqNum=5 because the exception is null or not the one we care about 2024-11-24T22:14:32,412 INFO [RS_FLUSH_OPERATIONS-regionserver/bd6592c37bb2:0-1 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/63842f73863e9e7b9587486b122e6aae/A/54b3fdfd1b984f089b1a2242f0d9d9b2, entries=150, sequenceid=174, filesize=30.4 K 2024-11-24T22:14:32,413 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bd6592c37bb2:0-1 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/63842f73863e9e7b9587486b122e6aae/.tmp/B/8546de4e512a4074a0c26e2f30373177 as hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/63842f73863e9e7b9587486b122e6aae/B/8546de4e512a4074a0c26e2f30373177 2024-11-24T22:14:32,420 INFO [RS_FLUSH_OPERATIONS-regionserver/bd6592c37bb2:0-1 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/63842f73863e9e7b9587486b122e6aae/B/8546de4e512a4074a0c26e2f30373177, entries=150, sequenceid=174, filesize=11.9 K 2024-11-24T22:14:32,420 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40087 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=63842f73863e9e7b9587486b122e6aae, server=bd6592c37bb2,40087,1732486423713 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-24T22:14:32,421 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40087 {}] ipc.CallRunner(138): callId: 66 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40244 deadline: 1732486532411, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=63842f73863e9e7b9587486b122e6aae, server=bd6592c37bb2,40087,1732486423713 2024-11-24T22:14:32,421 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732486460633.63842f73863e9e7b9587486b122e6aae., hostname=bd6592c37bb2,40087,1732486423713, seqNum=5 , the old value is region=TestAcidGuarantees,,1732486460633.63842f73863e9e7b9587486b122e6aae., hostname=bd6592c37bb2,40087,1732486423713, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=63842f73863e9e7b9587486b122e6aae, server=bd6592c37bb2,40087,1732486423713 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T22:14:32,421 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732486460633.63842f73863e9e7b9587486b122e6aae., hostname=bd6592c37bb2,40087,1732486423713, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=63842f73863e9e7b9587486b122e6aae, server=bd6592c37bb2,40087,1732486423713 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T22:14:32,421 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732486460633.63842f73863e9e7b9587486b122e6aae., hostname=bd6592c37bb2,40087,1732486423713, seqNum=5 because the exception is null or not the one we care about 2024-11-24T22:14:32,424 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bd6592c37bb2:0-1 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/63842f73863e9e7b9587486b122e6aae/.tmp/C/5c1e349f365f4cfda43b21302f209dec as hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/63842f73863e9e7b9587486b122e6aae/C/5c1e349f365f4cfda43b21302f209dec 2024-11-24T22:14:32,430 INFO [RS_FLUSH_OPERATIONS-regionserver/bd6592c37bb2:0-1 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/63842f73863e9e7b9587486b122e6aae/C/5c1e349f365f4cfda43b21302f209dec, entries=150, sequenceid=174, filesize=11.9 K 2024-11-24T22:14:32,431 INFO [RS_FLUSH_OPERATIONS-regionserver/bd6592c37bb2:0-1 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.HRegion(3140): Finished flush of dataSize ~87.22 KB/89310, heapSize ~259.69 KB/265920, currentSize=87.22 KB/89310 for 63842f73863e9e7b9587486b122e6aae in 1365ms, sequenceid=174, compaction requested=false 2024-11-24T22:14:32,431 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bd6592c37bb2:0-1 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.HRegion(2603): Flush status journal for 63842f73863e9e7b9587486b122e6aae: 2024-11-24T22:14:32,431 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bd6592c37bb2:0-1 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732486460633.63842f73863e9e7b9587486b122e6aae. 2024-11-24T22:14:32,431 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bd6592c37bb2:0-1 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=43 2024-11-24T22:14:32,431 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37701 {}] master.HMaster(4169): Remote procedure done, pid=43 2024-11-24T22:14:32,435 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=43, resume processing ppid=42 2024-11-24T22:14:32,435 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=43, ppid=42, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 2.2970 sec 2024-11-24T22:14:32,436 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=42, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=42, table=TestAcidGuarantees in 2.3030 sec 2024-11-24T22:14:33,425 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 63842f73863e9e7b9587486b122e6aae 3/3 column families, dataSize=93.93 KB heapSize=279.66 KB 2024-11-24T22:14:33,426 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 63842f73863e9e7b9587486b122e6aae, store=A 2024-11-24T22:14:33,426 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-24T22:14:33,426 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 63842f73863e9e7b9587486b122e6aae, store=B 2024-11-24T22:14:33,426 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-24T22:14:33,426 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 63842f73863e9e7b9587486b122e6aae, store=C 2024-11-24T22:14:33,426 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-24T22:14:33,427 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40087 {}] regionserver.HRegion(8855): Flush requested on 63842f73863e9e7b9587486b122e6aae 2024-11-24T22:14:33,469 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024112428689bd4df0c47fc925b82ed3bc805e2_63842f73863e9e7b9587486b122e6aae is 50, key is test_row_0/A:col10/1732486471252/Put/seqid=0 2024-11-24T22:14:33,489 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40087 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=63842f73863e9e7b9587486b122e6aae, server=bd6592c37bb2,40087,1732486423713 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-24T22:14:33,489 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40087 {}] ipc.CallRunner(138): callId: 73 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40244 deadline: 1732486533481, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=63842f73863e9e7b9587486b122e6aae, server=bd6592c37bb2,40087,1732486423713 2024-11-24T22:14:33,489 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732486460633.63842f73863e9e7b9587486b122e6aae., hostname=bd6592c37bb2,40087,1732486423713, seqNum=5 , the old value is region=TestAcidGuarantees,,1732486460633.63842f73863e9e7b9587486b122e6aae., hostname=bd6592c37bb2,40087,1732486423713, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=63842f73863e9e7b9587486b122e6aae, server=bd6592c37bb2,40087,1732486423713 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T22:14:33,490 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732486460633.63842f73863e9e7b9587486b122e6aae., hostname=bd6592c37bb2,40087,1732486423713, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=63842f73863e9e7b9587486b122e6aae, server=bd6592c37bb2,40087,1732486423713 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T22:14:33,490 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732486460633.63842f73863e9e7b9587486b122e6aae., hostname=bd6592c37bb2,40087,1732486423713, seqNum=5 because the exception is null or not the one we care about 2024-11-24T22:14:33,490 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40087 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=63842f73863e9e7b9587486b122e6aae, server=bd6592c37bb2,40087,1732486423713 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-24T22:14:33,490 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40087 {}] ipc.CallRunner(138): callId: 82 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40206 deadline: 1732486533485, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=63842f73863e9e7b9587486b122e6aae, server=bd6592c37bb2,40087,1732486423713 2024-11-24T22:14:33,491 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732486460633.63842f73863e9e7b9587486b122e6aae., hostname=bd6592c37bb2,40087,1732486423713, seqNum=5 , the old value is region=TestAcidGuarantees,,1732486460633.63842f73863e9e7b9587486b122e6aae., hostname=bd6592c37bb2,40087,1732486423713, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=63842f73863e9e7b9587486b122e6aae, server=bd6592c37bb2,40087,1732486423713 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T22:14:33,491 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732486460633.63842f73863e9e7b9587486b122e6aae., hostname=bd6592c37bb2,40087,1732486423713, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=63842f73863e9e7b9587486b122e6aae, server=bd6592c37bb2,40087,1732486423713 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T22:14:33,491 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732486460633.63842f73863e9e7b9587486b122e6aae., hostname=bd6592c37bb2,40087,1732486423713, seqNum=5 because the exception is null or not the one we care about 2024-11-24T22:14:33,506 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45929 is added to blk_1073741987_1163 (size=12304) 2024-11-24T22:14:33,606 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40087 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=63842f73863e9e7b9587486b122e6aae, server=bd6592c37bb2,40087,1732486423713 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-24T22:14:33,607 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40087 {}] ipc.CallRunner(138): callId: 74 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40244 deadline: 1732486533601, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=63842f73863e9e7b9587486b122e6aae, server=bd6592c37bb2,40087,1732486423713 2024-11-24T22:14:33,607 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40087 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=63842f73863e9e7b9587486b122e6aae, server=bd6592c37bb2,40087,1732486423713 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-24T22:14:33,607 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40087 {}] ipc.CallRunner(138): callId: 83 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40206 deadline: 1732486533601, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=63842f73863e9e7b9587486b122e6aae, server=bd6592c37bb2,40087,1732486423713 2024-11-24T22:14:33,607 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732486460633.63842f73863e9e7b9587486b122e6aae., hostname=bd6592c37bb2,40087,1732486423713, seqNum=5 , the old value is region=TestAcidGuarantees,,1732486460633.63842f73863e9e7b9587486b122e6aae., hostname=bd6592c37bb2,40087,1732486423713, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=63842f73863e9e7b9587486b122e6aae, server=bd6592c37bb2,40087,1732486423713 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T22:14:33,607 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732486460633.63842f73863e9e7b9587486b122e6aae., hostname=bd6592c37bb2,40087,1732486423713, seqNum=5 , the old value is region=TestAcidGuarantees,,1732486460633.63842f73863e9e7b9587486b122e6aae., hostname=bd6592c37bb2,40087,1732486423713, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=63842f73863e9e7b9587486b122e6aae, server=bd6592c37bb2,40087,1732486423713 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T22:14:33,608 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732486460633.63842f73863e9e7b9587486b122e6aae., hostname=bd6592c37bb2,40087,1732486423713, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=63842f73863e9e7b9587486b122e6aae, server=bd6592c37bb2,40087,1732486423713 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T22:14:33,608 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732486460633.63842f73863e9e7b9587486b122e6aae., hostname=bd6592c37bb2,40087,1732486423713, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=63842f73863e9e7b9587486b122e6aae, server=bd6592c37bb2,40087,1732486423713 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T22:14:33,608 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732486460633.63842f73863e9e7b9587486b122e6aae., hostname=bd6592c37bb2,40087,1732486423713, seqNum=5 because the exception is null or not the one we care about 2024-11-24T22:14:33,608 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732486460633.63842f73863e9e7b9587486b122e6aae., hostname=bd6592c37bb2,40087,1732486423713, seqNum=5 because the exception is null or not the one we care about 2024-11-24T22:14:33,814 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40087 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=63842f73863e9e7b9587486b122e6aae, server=bd6592c37bb2,40087,1732486423713 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-24T22:14:33,814 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40087 {}] ipc.CallRunner(138): callId: 75 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40244 deadline: 1732486533811, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=63842f73863e9e7b9587486b122e6aae, server=bd6592c37bb2,40087,1732486423713 2024-11-24T22:14:33,815 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732486460633.63842f73863e9e7b9587486b122e6aae., hostname=bd6592c37bb2,40087,1732486423713, seqNum=5 , the old value is region=TestAcidGuarantees,,1732486460633.63842f73863e9e7b9587486b122e6aae., hostname=bd6592c37bb2,40087,1732486423713, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=63842f73863e9e7b9587486b122e6aae, server=bd6592c37bb2,40087,1732486423713 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T22:14:33,815 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732486460633.63842f73863e9e7b9587486b122e6aae., hostname=bd6592c37bb2,40087,1732486423713, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=63842f73863e9e7b9587486b122e6aae, server=bd6592c37bb2,40087,1732486423713 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T22:14:33,815 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732486460633.63842f73863e9e7b9587486b122e6aae., hostname=bd6592c37bb2,40087,1732486423713, seqNum=5 because the exception is null or not the one we care about 2024-11-24T22:14:33,815 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40087 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=63842f73863e9e7b9587486b122e6aae, server=bd6592c37bb2,40087,1732486423713 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-24T22:14:33,815 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40087 {}] ipc.CallRunner(138): callId: 84 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40206 deadline: 1732486533811, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=63842f73863e9e7b9587486b122e6aae, server=bd6592c37bb2,40087,1732486423713 2024-11-24T22:14:33,815 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732486460633.63842f73863e9e7b9587486b122e6aae., hostname=bd6592c37bb2,40087,1732486423713, seqNum=5 , the old value is region=TestAcidGuarantees,,1732486460633.63842f73863e9e7b9587486b122e6aae., hostname=bd6592c37bb2,40087,1732486423713, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=63842f73863e9e7b9587486b122e6aae, server=bd6592c37bb2,40087,1732486423713 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T22:14:33,816 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732486460633.63842f73863e9e7b9587486b122e6aae., hostname=bd6592c37bb2,40087,1732486423713, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=63842f73863e9e7b9587486b122e6aae, server=bd6592c37bb2,40087,1732486423713 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T22:14:33,816 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732486460633.63842f73863e9e7b9587486b122e6aae., hostname=bd6592c37bb2,40087,1732486423713, seqNum=5 because the exception is null or not the one we care about 2024-11-24T22:14:33,908 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T22:14:33,917 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024112428689bd4df0c47fc925b82ed3bc805e2_63842f73863e9e7b9587486b122e6aae to hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024112428689bd4df0c47fc925b82ed3bc805e2_63842f73863e9e7b9587486b122e6aae 2024-11-24T22:14:33,921 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/63842f73863e9e7b9587486b122e6aae/.tmp/A/1f159989c0a341c9990fdadf792c01a8, store: [table=TestAcidGuarantees family=A region=63842f73863e9e7b9587486b122e6aae] 2024-11-24T22:14:33,921 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/63842f73863e9e7b9587486b122e6aae/.tmp/A/1f159989c0a341c9990fdadf792c01a8 is 175, key is test_row_0/A:col10/1732486471252/Put/seqid=0 2024-11-24T22:14:33,970 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45929 is added to blk_1073741988_1164 (size=31105) 2024-11-24T22:14:33,971 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=194, memsize=31.3 K, hasBloomFilter=true, into tmp file hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/63842f73863e9e7b9587486b122e6aae/.tmp/A/1f159989c0a341c9990fdadf792c01a8 2024-11-24T22:14:33,996 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/63842f73863e9e7b9587486b122e6aae/.tmp/B/df1cfca44f444160a40eb44a80dd877f is 50, key is test_row_0/B:col10/1732486471252/Put/seqid=0 2024-11-24T22:14:34,035 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45929 is added to blk_1073741989_1165 (size=12151) 2024-11-24T22:14:34,126 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40087 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=63842f73863e9e7b9587486b122e6aae, server=bd6592c37bb2,40087,1732486423713 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-24T22:14:34,126 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40087 {}] ipc.CallRunner(138): callId: 85 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40206 deadline: 1732486534121, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=63842f73863e9e7b9587486b122e6aae, server=bd6592c37bb2,40087,1732486423713 2024-11-24T22:14:34,127 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732486460633.63842f73863e9e7b9587486b122e6aae., hostname=bd6592c37bb2,40087,1732486423713, seqNum=5 , the old value is region=TestAcidGuarantees,,1732486460633.63842f73863e9e7b9587486b122e6aae., hostname=bd6592c37bb2,40087,1732486423713, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=63842f73863e9e7b9587486b122e6aae, server=bd6592c37bb2,40087,1732486423713 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T22:14:34,127 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732486460633.63842f73863e9e7b9587486b122e6aae., hostname=bd6592c37bb2,40087,1732486423713, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=63842f73863e9e7b9587486b122e6aae, server=bd6592c37bb2,40087,1732486423713 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T22:14:34,127 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732486460633.63842f73863e9e7b9587486b122e6aae., hostname=bd6592c37bb2,40087,1732486423713, seqNum=5 because the exception is null or not the one we care about 2024-11-24T22:14:34,127 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40087 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=63842f73863e9e7b9587486b122e6aae, server=bd6592c37bb2,40087,1732486423713 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-24T22:14:34,128 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40087 {}] ipc.CallRunner(138): callId: 76 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40244 deadline: 1732486534121, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=63842f73863e9e7b9587486b122e6aae, server=bd6592c37bb2,40087,1732486423713 2024-11-24T22:14:34,128 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732486460633.63842f73863e9e7b9587486b122e6aae., hostname=bd6592c37bb2,40087,1732486423713, seqNum=5 , the old value is region=TestAcidGuarantees,,1732486460633.63842f73863e9e7b9587486b122e6aae., hostname=bd6592c37bb2,40087,1732486423713, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=63842f73863e9e7b9587486b122e6aae, server=bd6592c37bb2,40087,1732486423713 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T22:14:34,128 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732486460633.63842f73863e9e7b9587486b122e6aae., hostname=bd6592c37bb2,40087,1732486423713, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=63842f73863e9e7b9587486b122e6aae, server=bd6592c37bb2,40087,1732486423713 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T22:14:34,128 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732486460633.63842f73863e9e7b9587486b122e6aae., hostname=bd6592c37bb2,40087,1732486423713, seqNum=5 because the exception is null or not the one we care about 2024-11-24T22:14:34,301 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37701 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=42 2024-11-24T22:14:34,302 INFO [RPCClient-NioEventLoopGroup-4-7 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-24T22:14:34,303 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37701 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-24T22:14:34,306 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37701 {}] procedure2.ProcedureExecutor(1139): Stored pid=44, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=44, table=TestAcidGuarantees 2024-11-24T22:14:34,307 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=44, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=44, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-24T22:14:34,307 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37701 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=44 2024-11-24T22:14:34,308 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=44, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=44, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-24T22:14:34,308 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=45, ppid=44, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-24T22:14:34,312 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40087 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=63842f73863e9e7b9587486b122e6aae, server=bd6592c37bb2,40087,1732486423713 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-24T22:14:34,313 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40087 {}] ipc.CallRunner(138): callId: 50 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40202 deadline: 1732486534301, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=63842f73863e9e7b9587486b122e6aae, server=bd6592c37bb2,40087,1732486423713 2024-11-24T22:14:34,313 WARN [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRpcRetryingCaller(177): Call to bd6592c37bb2,40087,1732486423713 for 'test_row_0' in 63842f73863e9e7b9587486b122e6aae of TestAcidGuarantees failed, tries = 7, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 4198 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=63842f73863e9e7b9587486b122e6aae, server=bd6592c37bb2,40087,1732486423713 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor44.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=63842f73863e9e7b9587486b122e6aae, server=bd6592c37bb2,40087,1732486423713 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-24T22:14:34,314 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732486460633.63842f73863e9e7b9587486b122e6aae., hostname=bd6592c37bb2,40087,1732486423713, seqNum=5 , the old value is region=TestAcidGuarantees,,1732486460633.63842f73863e9e7b9587486b122e6aae., hostname=bd6592c37bb2,40087,1732486423713, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=63842f73863e9e7b9587486b122e6aae, server=bd6592c37bb2,40087,1732486423713 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T22:14:34,314 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732486460633.63842f73863e9e7b9587486b122e6aae., hostname=bd6592c37bb2,40087,1732486423713, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=63842f73863e9e7b9587486b122e6aae, server=bd6592c37bb2,40087,1732486423713 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T22:14:34,314 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732486460633.63842f73863e9e7b9587486b122e6aae., hostname=bd6592c37bb2,40087,1732486423713, seqNum=5 because the exception is null or not the one we care about 2024-11-24T22:14:34,411 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37701 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=44 2024-11-24T22:14:34,436 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=31.31 KB at sequenceid=194 (bloomFilter=true), to=hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/63842f73863e9e7b9587486b122e6aae/.tmp/B/df1cfca44f444160a40eb44a80dd877f 2024-11-24T22:14:34,445 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/63842f73863e9e7b9587486b122e6aae/.tmp/C/9d3c116f95e046b89d2b1d00eae6999d is 50, key is test_row_0/C:col10/1732486471252/Put/seqid=0 2024-11-24T22:14:34,449 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45929 is added to blk_1073741990_1166 (size=12151) 2024-11-24T22:14:34,462 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=40087 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=45 2024-11-24T22:14:34,462 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bd6592c37bb2:0-2 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732486460633.63842f73863e9e7b9587486b122e6aae. 2024-11-24T22:14:34,462 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bd6592c37bb2:0-2 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732486460633.63842f73863e9e7b9587486b122e6aae. as already flushing 2024-11-24T22:14:34,462 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bd6592c37bb2:0-2 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732486460633.63842f73863e9e7b9587486b122e6aae. 2024-11-24T22:14:34,462 ERROR [RS_FLUSH_OPERATIONS-regionserver/bd6592c37bb2:0-2 {event_type=RS_FLUSH_REGIONS, pid=45}] handler.RSProcedureHandler(58): pid=45 java.io.IOException: Unable to complete flush {ENCODED => 63842f73863e9e7b9587486b122e6aae, NAME => 'TestAcidGuarantees,,1732486460633.63842f73863e9e7b9587486b122e6aae.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T22:14:34,462 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bd6592c37bb2:0-2 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=45 java.io.IOException: Unable to complete flush {ENCODED => 63842f73863e9e7b9587486b122e6aae, NAME => 'TestAcidGuarantees,,1732486460633.63842f73863e9e7b9587486b122e6aae.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T22:14:34,463 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37701 {}] master.HMaster(4177): Remote procedure failed, pid=45 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 63842f73863e9e7b9587486b122e6aae, NAME => 'TestAcidGuarantees,,1732486460633.63842f73863e9e7b9587486b122e6aae.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 63842f73863e9e7b9587486b122e6aae, NAME => 'TestAcidGuarantees,,1732486460633.63842f73863e9e7b9587486b122e6aae.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T22:14:34,615 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=40087 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=45 2024-11-24T22:14:34,615 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bd6592c37bb2:0-0 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732486460633.63842f73863e9e7b9587486b122e6aae. 2024-11-24T22:14:34,616 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bd6592c37bb2:0-0 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732486460633.63842f73863e9e7b9587486b122e6aae. as already flushing 2024-11-24T22:14:34,616 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bd6592c37bb2:0-0 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732486460633.63842f73863e9e7b9587486b122e6aae. 2024-11-24T22:14:34,616 ERROR [RS_FLUSH_OPERATIONS-regionserver/bd6592c37bb2:0-0 {event_type=RS_FLUSH_REGIONS, pid=45}] handler.RSProcedureHandler(58): pid=45 java.io.IOException: Unable to complete flush {ENCODED => 63842f73863e9e7b9587486b122e6aae, NAME => 'TestAcidGuarantees,,1732486460633.63842f73863e9e7b9587486b122e6aae.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T22:14:34,616 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bd6592c37bb2:0-0 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=45 java.io.IOException: Unable to complete flush {ENCODED => 63842f73863e9e7b9587486b122e6aae, NAME => 'TestAcidGuarantees,,1732486460633.63842f73863e9e7b9587486b122e6aae.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T22:14:34,616 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37701 {}] master.HMaster(4177): Remote procedure failed, pid=45 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 63842f73863e9e7b9587486b122e6aae, NAME => 'TestAcidGuarantees,,1732486460633.63842f73863e9e7b9587486b122e6aae.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 63842f73863e9e7b9587486b122e6aae, NAME => 'TestAcidGuarantees,,1732486460633.63842f73863e9e7b9587486b122e6aae.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T22:14:34,621 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37701 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=44 2024-11-24T22:14:34,633 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40087 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=63842f73863e9e7b9587486b122e6aae, server=bd6592c37bb2,40087,1732486423713 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-24T22:14:34,633 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40087 {}] ipc.CallRunner(138): callId: 86 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40206 deadline: 1732486534631, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=63842f73863e9e7b9587486b122e6aae, server=bd6592c37bb2,40087,1732486423713 2024-11-24T22:14:34,634 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732486460633.63842f73863e9e7b9587486b122e6aae., hostname=bd6592c37bb2,40087,1732486423713, seqNum=5 , the old value is region=TestAcidGuarantees,,1732486460633.63842f73863e9e7b9587486b122e6aae., hostname=bd6592c37bb2,40087,1732486423713, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=63842f73863e9e7b9587486b122e6aae, server=bd6592c37bb2,40087,1732486423713 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T22:14:34,634 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732486460633.63842f73863e9e7b9587486b122e6aae., hostname=bd6592c37bb2,40087,1732486423713, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=63842f73863e9e7b9587486b122e6aae, server=bd6592c37bb2,40087,1732486423713 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T22:14:34,634 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732486460633.63842f73863e9e7b9587486b122e6aae., hostname=bd6592c37bb2,40087,1732486423713, seqNum=5 because the exception is null or not the one we care about 2024-11-24T22:14:34,646 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40087 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=63842f73863e9e7b9587486b122e6aae, server=bd6592c37bb2,40087,1732486423713 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-24T22:14:34,647 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40087 {}] ipc.CallRunner(138): callId: 77 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40244 deadline: 1732486534640, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=63842f73863e9e7b9587486b122e6aae, server=bd6592c37bb2,40087,1732486423713 2024-11-24T22:14:34,647 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732486460633.63842f73863e9e7b9587486b122e6aae., hostname=bd6592c37bb2,40087,1732486423713, seqNum=5 , the old value is region=TestAcidGuarantees,,1732486460633.63842f73863e9e7b9587486b122e6aae., hostname=bd6592c37bb2,40087,1732486423713, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=63842f73863e9e7b9587486b122e6aae, server=bd6592c37bb2,40087,1732486423713 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T22:14:34,647 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732486460633.63842f73863e9e7b9587486b122e6aae., hostname=bd6592c37bb2,40087,1732486423713, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=63842f73863e9e7b9587486b122e6aae, server=bd6592c37bb2,40087,1732486423713 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T22:14:34,647 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732486460633.63842f73863e9e7b9587486b122e6aae., hostname=bd6592c37bb2,40087,1732486423713, seqNum=5 because the exception is null or not the one we care about 2024-11-24T22:14:34,769 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=40087 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=45 2024-11-24T22:14:34,769 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bd6592c37bb2:0-1 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732486460633.63842f73863e9e7b9587486b122e6aae. 2024-11-24T22:14:34,769 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bd6592c37bb2:0-1 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732486460633.63842f73863e9e7b9587486b122e6aae. as already flushing 2024-11-24T22:14:34,769 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bd6592c37bb2:0-1 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732486460633.63842f73863e9e7b9587486b122e6aae. 2024-11-24T22:14:34,770 ERROR [RS_FLUSH_OPERATIONS-regionserver/bd6592c37bb2:0-1 {event_type=RS_FLUSH_REGIONS, pid=45}] handler.RSProcedureHandler(58): pid=45 java.io.IOException: Unable to complete flush {ENCODED => 63842f73863e9e7b9587486b122e6aae, NAME => 'TestAcidGuarantees,,1732486460633.63842f73863e9e7b9587486b122e6aae.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T22:14:34,770 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bd6592c37bb2:0-1 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=45 java.io.IOException: Unable to complete flush {ENCODED => 63842f73863e9e7b9587486b122e6aae, NAME => 'TestAcidGuarantees,,1732486460633.63842f73863e9e7b9587486b122e6aae.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T22:14:34,777 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37701 {}] master.HMaster(4177): Remote procedure failed, pid=45 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 63842f73863e9e7b9587486b122e6aae, NAME => 'TestAcidGuarantees,,1732486460633.63842f73863e9e7b9587486b122e6aae.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 63842f73863e9e7b9587486b122e6aae, NAME => 'TestAcidGuarantees,,1732486460633.63842f73863e9e7b9587486b122e6aae.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T22:14:34,850 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=31.31 KB at sequenceid=194 (bloomFilter=true), to=hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/63842f73863e9e7b9587486b122e6aae/.tmp/C/9d3c116f95e046b89d2b1d00eae6999d 2024-11-24T22:14:34,856 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/63842f73863e9e7b9587486b122e6aae/.tmp/A/1f159989c0a341c9990fdadf792c01a8 as hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/63842f73863e9e7b9587486b122e6aae/A/1f159989c0a341c9990fdadf792c01a8 2024-11-24T22:14:34,864 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/63842f73863e9e7b9587486b122e6aae/A/1f159989c0a341c9990fdadf792c01a8, entries=150, sequenceid=194, filesize=30.4 K 2024-11-24T22:14:34,866 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/63842f73863e9e7b9587486b122e6aae/.tmp/B/df1cfca44f444160a40eb44a80dd877f as hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/63842f73863e9e7b9587486b122e6aae/B/df1cfca44f444160a40eb44a80dd877f 2024-11-24T22:14:34,874 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/63842f73863e9e7b9587486b122e6aae/B/df1cfca44f444160a40eb44a80dd877f, entries=150, sequenceid=194, filesize=11.9 K 2024-11-24T22:14:34,876 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/63842f73863e9e7b9587486b122e6aae/.tmp/C/9d3c116f95e046b89d2b1d00eae6999d as hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/63842f73863e9e7b9587486b122e6aae/C/9d3c116f95e046b89d2b1d00eae6999d 2024-11-24T22:14:34,885 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/63842f73863e9e7b9587486b122e6aae/C/9d3c116f95e046b89d2b1d00eae6999d, entries=150, sequenceid=194, filesize=11.9 K 2024-11-24T22:14:34,887 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~93.93 KB/96180, heapSize ~279.61 KB/286320, currentSize=80.51 KB/82440 for 63842f73863e9e7b9587486b122e6aae in 1461ms, sequenceid=194, compaction requested=true 2024-11-24T22:14:34,887 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 63842f73863e9e7b9587486b122e6aae: 2024-11-24T22:14:34,887 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 63842f73863e9e7b9587486b122e6aae:A, priority=-2147483648, current under compaction store size is 1 2024-11-24T22:14:34,887 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-24T22:14:34,887 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 63842f73863e9e7b9587486b122e6aae:B, priority=-2147483648, current under compaction store size is 2 2024-11-24T22:14:34,887 DEBUG [RS:0;bd6592c37bb2:40087-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-24T22:14:34,887 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-24T22:14:34,887 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 63842f73863e9e7b9587486b122e6aae:C, priority=-2147483648, current under compaction store size is 3 2024-11-24T22:14:34,887 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=2), splitQueue=0 2024-11-24T22:14:34,887 DEBUG [RS:0;bd6592c37bb2:40087-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-24T22:14:34,891 DEBUG [RS:0;bd6592c37bb2:40087-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36795 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-24T22:14:34,891 DEBUG [RS:0;bd6592c37bb2:40087-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 93657 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-24T22:14:34,891 DEBUG [RS:0;bd6592c37bb2:40087-longCompactions-0 {}] regionserver.HStore(1541): 63842f73863e9e7b9587486b122e6aae/B is initiating minor compaction (all files) 2024-11-24T22:14:34,891 DEBUG [RS:0;bd6592c37bb2:40087-shortCompactions-0 {}] regionserver.HStore(1541): 63842f73863e9e7b9587486b122e6aae/A is initiating minor compaction (all files) 2024-11-24T22:14:34,891 INFO [RS:0;bd6592c37bb2:40087-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 63842f73863e9e7b9587486b122e6aae/B in TestAcidGuarantees,,1732486460633.63842f73863e9e7b9587486b122e6aae. 2024-11-24T22:14:34,892 INFO [RS:0;bd6592c37bb2:40087-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/63842f73863e9e7b9587486b122e6aae/B/ed25299c01a145d88eaa3f03fd0b147e, hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/63842f73863e9e7b9587486b122e6aae/B/8546de4e512a4074a0c26e2f30373177, hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/63842f73863e9e7b9587486b122e6aae/B/df1cfca44f444160a40eb44a80dd877f] into tmpdir=hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/63842f73863e9e7b9587486b122e6aae/.tmp, totalSize=35.9 K 2024-11-24T22:14:34,892 INFO [RS:0;bd6592c37bb2:40087-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 63842f73863e9e7b9587486b122e6aae/A in TestAcidGuarantees,,1732486460633.63842f73863e9e7b9587486b122e6aae. 2024-11-24T22:14:34,892 INFO [RS:0;bd6592c37bb2:40087-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/63842f73863e9e7b9587486b122e6aae/A/a4e17bb33ebb463ea0574c07bb661b48, hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/63842f73863e9e7b9587486b122e6aae/A/54b3fdfd1b984f089b1a2242f0d9d9b2, hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/63842f73863e9e7b9587486b122e6aae/A/1f159989c0a341c9990fdadf792c01a8] into tmpdir=hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/63842f73863e9e7b9587486b122e6aae/.tmp, totalSize=91.5 K 2024-11-24T22:14:34,892 INFO [RS:0;bd6592c37bb2:40087-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=13 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1732486460633.63842f73863e9e7b9587486b122e6aae. 2024-11-24T22:14:34,892 DEBUG [RS:0;bd6592c37bb2:40087-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1732486460633.63842f73863e9e7b9587486b122e6aae. files: [hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/63842f73863e9e7b9587486b122e6aae/A/a4e17bb33ebb463ea0574c07bb661b48, hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/63842f73863e9e7b9587486b122e6aae/A/54b3fdfd1b984f089b1a2242f0d9d9b2, hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/63842f73863e9e7b9587486b122e6aae/A/1f159989c0a341c9990fdadf792c01a8] 2024-11-24T22:14:34,892 DEBUG [RS:0;bd6592c37bb2:40087-longCompactions-0 {}] compactions.Compactor(225): Compacting ed25299c01a145d88eaa3f03fd0b147e, keycount=150, bloomtype=ROW, size=12.2 K, encoding=NONE, compression=NONE, seqNum=158, earliestPutTs=1732486469042 2024-11-24T22:14:34,893 DEBUG [RS:0;bd6592c37bb2:40087-shortCompactions-0 {}] compactions.Compactor(225): Compacting a4e17bb33ebb463ea0574c07bb661b48, keycount=150, bloomtype=ROW, size=30.7 K, encoding=NONE, compression=NONE, seqNum=158, earliestPutTs=1732486469042 2024-11-24T22:14:34,893 DEBUG [RS:0;bd6592c37bb2:40087-longCompactions-0 {}] compactions.Compactor(225): Compacting 8546de4e512a4074a0c26e2f30373177, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=174, earliestPutTs=1732486470080 2024-11-24T22:14:34,894 DEBUG [RS:0;bd6592c37bb2:40087-shortCompactions-0 {}] compactions.Compactor(225): Compacting 54b3fdfd1b984f089b1a2242f0d9d9b2, keycount=150, bloomtype=ROW, size=30.4 K, encoding=NONE, compression=NONE, seqNum=174, earliestPutTs=1732486470080 2024-11-24T22:14:34,894 DEBUG [RS:0;bd6592c37bb2:40087-longCompactions-0 {}] compactions.Compactor(225): Compacting df1cfca44f444160a40eb44a80dd877f, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=194, earliestPutTs=1732486471251 2024-11-24T22:14:34,894 DEBUG [RS:0;bd6592c37bb2:40087-shortCompactions-0 {}] compactions.Compactor(225): Compacting 1f159989c0a341c9990fdadf792c01a8, keycount=150, bloomtype=ROW, size=30.4 K, encoding=NONE, compression=NONE, seqNum=194, earliestPutTs=1732486471251 2024-11-24T22:14:34,916 INFO [RS:0;bd6592c37bb2:40087-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=63842f73863e9e7b9587486b122e6aae] 2024-11-24T22:14:34,931 INFO [RS:0;bd6592c37bb2:40087-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 63842f73863e9e7b9587486b122e6aae#B#compaction#139 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-24T22:14:34,931 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37701 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=44 2024-11-24T22:14:34,931 DEBUG [RS:0;bd6592c37bb2:40087-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/63842f73863e9e7b9587486b122e6aae/.tmp/B/7a02ad10845946b78864d3662caaedbd is 50, key is test_row_0/B:col10/1732486471252/Put/seqid=0 2024-11-24T22:14:34,937 DEBUG [RS:0;bd6592c37bb2:40087-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e202411240680a467fcf647a7981e842777f22add_63842f73863e9e7b9587486b122e6aae store=[table=TestAcidGuarantees family=A region=63842f73863e9e7b9587486b122e6aae] 2024-11-24T22:14:34,938 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=40087 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=45 2024-11-24T22:14:34,939 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bd6592c37bb2:0-2 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732486460633.63842f73863e9e7b9587486b122e6aae. 2024-11-24T22:14:34,939 INFO [RS_FLUSH_OPERATIONS-regionserver/bd6592c37bb2:0-2 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.HRegion(2902): Flushing 63842f73863e9e7b9587486b122e6aae 3/3 column families, dataSize=80.51 KB heapSize=239.81 KB 2024-11-24T22:14:34,939 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bd6592c37bb2:0-2 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 63842f73863e9e7b9587486b122e6aae, store=A 2024-11-24T22:14:34,939 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bd6592c37bb2:0-2 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-24T22:14:34,939 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bd6592c37bb2:0-2 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 63842f73863e9e7b9587486b122e6aae, store=B 2024-11-24T22:14:34,939 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bd6592c37bb2:0-2 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-24T22:14:34,939 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bd6592c37bb2:0-2 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 63842f73863e9e7b9587486b122e6aae, store=C 2024-11-24T22:14:34,939 DEBUG [RS:0;bd6592c37bb2:40087-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e202411240680a467fcf647a7981e842777f22add_63842f73863e9e7b9587486b122e6aae, store=[table=TestAcidGuarantees family=A region=63842f73863e9e7b9587486b122e6aae] 2024-11-24T22:14:34,939 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bd6592c37bb2:0-2 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-24T22:14:34,940 DEBUG [RS:0;bd6592c37bb2:40087-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411240680a467fcf647a7981e842777f22add_63842f73863e9e7b9587486b122e6aae because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=63842f73863e9e7b9587486b122e6aae] 2024-11-24T22:14:34,974 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45929 is added to blk_1073741991_1167 (size=12595) 2024-11-24T22:14:34,990 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bd6592c37bb2:0-2 {event_type=RS_FLUSH_REGIONS, pid=45}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241124a8603a69dd26464b9c54ecffa9c61a8c_63842f73863e9e7b9587486b122e6aae is 50, key is test_row_0/A:col10/1732486473479/Put/seqid=0 2024-11-24T22:14:34,997 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45929 is added to blk_1073741992_1168 (size=4469) 2024-11-24T22:14:35,030 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45929 is added to blk_1073741993_1169 (size=12304) 2024-11-24T22:14:35,381 DEBUG [RS:0;bd6592c37bb2:40087-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/63842f73863e9e7b9587486b122e6aae/.tmp/B/7a02ad10845946b78864d3662caaedbd as hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/63842f73863e9e7b9587486b122e6aae/B/7a02ad10845946b78864d3662caaedbd 2024-11-24T22:14:35,387 INFO [RS:0;bd6592c37bb2:40087-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 63842f73863e9e7b9587486b122e6aae/B of 63842f73863e9e7b9587486b122e6aae into 7a02ad10845946b78864d3662caaedbd(size=12.3 K), total size for store is 12.3 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-24T22:14:35,387 DEBUG [RS:0;bd6592c37bb2:40087-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 63842f73863e9e7b9587486b122e6aae: 2024-11-24T22:14:35,387 INFO [RS:0;bd6592c37bb2:40087-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732486460633.63842f73863e9e7b9587486b122e6aae., storeName=63842f73863e9e7b9587486b122e6aae/B, priority=13, startTime=1732486474887; duration=0sec 2024-11-24T22:14:35,387 DEBUG [RS:0;bd6592c37bb2:40087-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-24T22:14:35,387 DEBUG [RS:0;bd6592c37bb2:40087-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 63842f73863e9e7b9587486b122e6aae:B 2024-11-24T22:14:35,387 DEBUG [RS:0;bd6592c37bb2:40087-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-24T22:14:35,388 DEBUG [RS:0;bd6592c37bb2:40087-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36795 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-24T22:14:35,389 DEBUG [RS:0;bd6592c37bb2:40087-longCompactions-0 {}] regionserver.HStore(1541): 63842f73863e9e7b9587486b122e6aae/C is initiating minor compaction (all files) 2024-11-24T22:14:35,389 INFO [RS:0;bd6592c37bb2:40087-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 63842f73863e9e7b9587486b122e6aae/C in TestAcidGuarantees,,1732486460633.63842f73863e9e7b9587486b122e6aae. 2024-11-24T22:14:35,389 INFO [RS:0;bd6592c37bb2:40087-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/63842f73863e9e7b9587486b122e6aae/C/89494cbdc6fb4d5da656158a799b399e, hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/63842f73863e9e7b9587486b122e6aae/C/5c1e349f365f4cfda43b21302f209dec, hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/63842f73863e9e7b9587486b122e6aae/C/9d3c116f95e046b89d2b1d00eae6999d] into tmpdir=hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/63842f73863e9e7b9587486b122e6aae/.tmp, totalSize=35.9 K 2024-11-24T22:14:35,389 DEBUG [RS:0;bd6592c37bb2:40087-longCompactions-0 {}] compactions.Compactor(225): Compacting 89494cbdc6fb4d5da656158a799b399e, keycount=150, bloomtype=ROW, size=12.2 K, encoding=NONE, compression=NONE, seqNum=158, earliestPutTs=1732486469042 2024-11-24T22:14:35,390 DEBUG [RS:0;bd6592c37bb2:40087-longCompactions-0 {}] compactions.Compactor(225): Compacting 5c1e349f365f4cfda43b21302f209dec, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=174, earliestPutTs=1732486470080 2024-11-24T22:14:35,390 DEBUG [RS:0;bd6592c37bb2:40087-longCompactions-0 {}] compactions.Compactor(225): Compacting 9d3c116f95e046b89d2b1d00eae6999d, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=194, earliestPutTs=1732486471251 2024-11-24T22:14:35,397 INFO [RS:0;bd6592c37bb2:40087-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 63842f73863e9e7b9587486b122e6aae#C#compaction#141 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-24T22:14:35,397 DEBUG [RS:0;bd6592c37bb2:40087-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/63842f73863e9e7b9587486b122e6aae/.tmp/C/4923864c6fda45d6b701d9e800bd4c46 is 50, key is test_row_0/C:col10/1732486471252/Put/seqid=0 2024-11-24T22:14:35,399 INFO [RS:0;bd6592c37bb2:40087-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 63842f73863e9e7b9587486b122e6aae#A#compaction#138 average throughput is 0.05 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-24T22:14:35,400 DEBUG [RS:0;bd6592c37bb2:40087-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/63842f73863e9e7b9587486b122e6aae/.tmp/A/937c2f75b1e34c80aa8852008cd4dce4 is 175, key is test_row_0/A:col10/1732486471252/Put/seqid=0 2024-11-24T22:14:35,413 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45929 is added to blk_1073741994_1170 (size=12595) 2024-11-24T22:14:35,419 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45929 is added to blk_1073741995_1171 (size=31549) 2024-11-24T22:14:35,433 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bd6592c37bb2:0-2 {event_type=RS_FLUSH_REGIONS, pid=45}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T22:14:35,439 INFO [RS_FLUSH_OPERATIONS-regionserver/bd6592c37bb2:0-2 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241124a8603a69dd26464b9c54ecffa9c61a8c_63842f73863e9e7b9587486b122e6aae to hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241124a8603a69dd26464b9c54ecffa9c61a8c_63842f73863e9e7b9587486b122e6aae 2024-11-24T22:14:35,441 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37701 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=44 2024-11-24T22:14:35,441 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bd6592c37bb2:0-2 {event_type=RS_FLUSH_REGIONS, pid=45}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/63842f73863e9e7b9587486b122e6aae/.tmp/A/a80b645125754d9299915c8d52dab903, store: [table=TestAcidGuarantees family=A region=63842f73863e9e7b9587486b122e6aae] 2024-11-24T22:14:35,441 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bd6592c37bb2:0-2 {event_type=RS_FLUSH_REGIONS, pid=45}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/63842f73863e9e7b9587486b122e6aae/.tmp/A/a80b645125754d9299915c8d52dab903 is 175, key is test_row_0/A:col10/1732486473479/Put/seqid=0 2024-11-24T22:14:35,450 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45929 is added to blk_1073741996_1172 (size=31105) 2024-11-24T22:14:35,451 INFO [RS_FLUSH_OPERATIONS-regionserver/bd6592c37bb2:0-2 {event_type=RS_FLUSH_REGIONS, pid=45}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=209, memsize=26.8 K, hasBloomFilter=true, into tmp file hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/63842f73863e9e7b9587486b122e6aae/.tmp/A/a80b645125754d9299915c8d52dab903 2024-11-24T22:14:35,480 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bd6592c37bb2:0-2 {event_type=RS_FLUSH_REGIONS, pid=45}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/63842f73863e9e7b9587486b122e6aae/.tmp/B/80c3be87a7bc406abcd49b8ff894561f is 50, key is test_row_0/B:col10/1732486473479/Put/seqid=0 2024-11-24T22:14:35,504 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45929 is added to blk_1073741997_1173 (size=12151) 2024-11-24T22:14:35,507 INFO [RS_FLUSH_OPERATIONS-regionserver/bd6592c37bb2:0-2 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=26.84 KB at sequenceid=209 (bloomFilter=true), to=hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/63842f73863e9e7b9587486b122e6aae/.tmp/B/80c3be87a7bc406abcd49b8ff894561f 2024-11-24T22:14:35,521 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bd6592c37bb2:0-2 {event_type=RS_FLUSH_REGIONS, pid=45}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/63842f73863e9e7b9587486b122e6aae/.tmp/C/58561c0c991f467ca37137ea8d0cf3df is 50, key is test_row_0/C:col10/1732486473479/Put/seqid=0 2024-11-24T22:14:35,534 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45929 is added to blk_1073741998_1174 (size=12151) 2024-11-24T22:14:35,534 INFO [RS_FLUSH_OPERATIONS-regionserver/bd6592c37bb2:0-2 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=26.84 KB at sequenceid=209 (bloomFilter=true), to=hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/63842f73863e9e7b9587486b122e6aae/.tmp/C/58561c0c991f467ca37137ea8d0cf3df 2024-11-24T22:14:35,541 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bd6592c37bb2:0-2 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/63842f73863e9e7b9587486b122e6aae/.tmp/A/a80b645125754d9299915c8d52dab903 as hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/63842f73863e9e7b9587486b122e6aae/A/a80b645125754d9299915c8d52dab903 2024-11-24T22:14:35,546 INFO [RS_FLUSH_OPERATIONS-regionserver/bd6592c37bb2:0-2 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/63842f73863e9e7b9587486b122e6aae/A/a80b645125754d9299915c8d52dab903, entries=150, sequenceid=209, filesize=30.4 K 2024-11-24T22:14:35,546 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bd6592c37bb2:0-2 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.StoreScanner(997): StoreScanner already has the close lock. There is no need to updateReaders 2024-11-24T22:14:35,548 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bd6592c37bb2:0-2 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/63842f73863e9e7b9587486b122e6aae/.tmp/B/80c3be87a7bc406abcd49b8ff894561f as hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/63842f73863e9e7b9587486b122e6aae/B/80c3be87a7bc406abcd49b8ff894561f 2024-11-24T22:14:35,551 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T22:14:35,551 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T22:14:35,551 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T22:14:35,554 INFO [RS_FLUSH_OPERATIONS-regionserver/bd6592c37bb2:0-2 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/63842f73863e9e7b9587486b122e6aae/B/80c3be87a7bc406abcd49b8ff894561f, entries=150, sequenceid=209, filesize=11.9 K 2024-11-24T22:14:35,555 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bd6592c37bb2:0-2 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/63842f73863e9e7b9587486b122e6aae/.tmp/C/58561c0c991f467ca37137ea8d0cf3df as hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/63842f73863e9e7b9587486b122e6aae/C/58561c0c991f467ca37137ea8d0cf3df 2024-11-24T22:14:35,560 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T22:14:35,560 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T22:14:35,560 INFO [RS_FLUSH_OPERATIONS-regionserver/bd6592c37bb2:0-2 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/63842f73863e9e7b9587486b122e6aae/C/58561c0c991f467ca37137ea8d0cf3df, entries=150, sequenceid=209, filesize=11.9 K 2024-11-24T22:14:35,561 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T22:14:35,561 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T22:14:35,561 INFO [RS_FLUSH_OPERATIONS-regionserver/bd6592c37bb2:0-2 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.HRegion(3140): Finished flush of dataSize ~80.51 KB/82440, heapSize ~239.77 KB/245520, currentSize=0 B/0 for 63842f73863e9e7b9587486b122e6aae in 622ms, sequenceid=209, compaction requested=false 2024-11-24T22:14:35,561 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bd6592c37bb2:0-2 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.HRegion(2603): Flush status journal for 63842f73863e9e7b9587486b122e6aae: 2024-11-24T22:14:35,561 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bd6592c37bb2:0-2 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732486460633.63842f73863e9e7b9587486b122e6aae. 2024-11-24T22:14:35,561 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bd6592c37bb2:0-2 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=45 2024-11-24T22:14:35,562 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T22:14:35,562 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T22:14:35,562 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37701 {}] master.HMaster(4169): Remote procedure done, pid=45 2024-11-24T22:14:35,563 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T22:14:35,563 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T22:14:35,563 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T22:14:35,564 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T22:14:35,565 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T22:14:35,565 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T22:14:35,565 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=45, resume processing ppid=44 2024-11-24T22:14:35,565 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=45, ppid=44, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.2550 sec 2024-11-24T22:14:35,566 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T22:14:35,566 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T22:14:35,567 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T22:14:35,567 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=44, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=44, table=TestAcidGuarantees in 1.2630 sec 2024-11-24T22:14:35,569 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T22:14:35,570 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T22:14:35,571 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T22:14:35,573 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T22:14:35,574 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T22:14:35,575 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T22:14:35,576 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T22:14:35,576 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T22:14:35,576 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T22:14:35,577 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T22:14:35,577 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T22:14:35,578 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T22:14:35,578 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T22:14:35,578 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T22:14:35,579 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T22:14:35,579 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T22:14:35,580 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T22:14:35,580 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T22:14:35,581 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T22:14:35,581 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T22:14:35,582 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T22:14:35,582 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T22:14:35,582 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T22:14:35,583 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T22:14:35,583 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T22:14:35,583 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T22:14:35,583 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T22:14:35,584 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T22:14:35,584 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T22:14:35,584 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T22:14:35,585 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T22:14:35,585 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T22:14:35,585 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T22:14:35,585 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T22:14:35,586 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T22:14:35,586 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T22:14:35,586 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T22:14:35,587 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T22:14:35,587 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T22:14:35,587 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T22:14:35,587 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T22:14:35,588 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T22:14:35,588 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T22:14:35,588 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T22:14:35,588 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T22:14:35,588 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T22:14:35,589 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T22:14:35,589 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T22:14:35,589 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T22:14:35,590 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T22:14:35,590 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T22:14:35,591 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T22:14:35,591 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T22:14:35,591 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T22:14:35,591 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T22:14:35,592 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T22:14:35,592 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T22:14:35,592 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T22:14:35,592 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T22:14:35,593 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T22:14:35,593 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T22:14:35,593 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T22:14:35,593 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T22:14:35,593 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T22:14:35,594 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T22:14:35,594 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T22:14:35,594 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T22:14:35,594 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T22:14:35,595 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T22:14:35,595 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T22:14:35,595 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T22:14:35,596 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T22:14:35,596 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T22:14:35,597 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T22:14:35,597 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T22:14:35,597 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T22:14:35,597 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T22:14:35,598 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T22:14:35,598 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T22:14:35,599 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T22:14:35,599 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T22:14:35,599 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T22:14:35,599 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T22:14:35,600 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T22:14:35,601 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T22:14:35,601 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T22:14:35,601 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T22:14:35,602 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T22:14:35,602 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T22:14:35,602 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T22:14:35,603 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T22:14:35,603 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T22:14:35,603 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T22:14:35,604 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T22:14:35,604 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T22:14:35,604 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T22:14:35,605 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T22:14:35,605 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T22:14:35,606 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T22:14:35,606 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T22:14:35,606 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T22:14:35,606 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T22:14:35,607 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T22:14:35,607 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T22:14:35,607 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T22:14:35,608 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T22:14:35,608 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T22:14:35,609 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T22:14:35,609 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T22:14:35,609 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T22:14:35,610 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T22:14:35,610 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T22:14:35,610 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T22:14:35,611 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T22:14:35,611 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T22:14:35,612 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T22:14:35,612 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T22:14:35,613 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T22:14:35,614 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T22:14:35,615 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T22:14:35,615 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T22:14:35,616 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T22:14:35,616 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T22:14:35,617 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T22:14:35,617 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T22:14:35,617 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T22:14:35,617 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T22:14:35,618 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T22:14:35,618 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T22:14:35,619 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T22:14:35,619 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T22:14:35,619 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T22:14:35,619 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T22:14:35,619 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T22:14:35,620 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T22:14:35,620 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T22:14:35,620 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T22:14:35,621 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T22:14:35,621 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T22:14:35,621 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T22:14:35,621 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T22:14:35,622 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T22:14:35,622 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T22:14:35,622 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T22:14:35,622 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T22:14:35,622 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T22:14:35,623 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T22:14:35,623 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T22:14:35,623 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T22:14:35,623 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T22:14:35,624 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T22:14:35,624 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T22:14:35,624 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T22:14:35,625 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T22:14:35,625 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T22:14:35,625 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T22:14:35,626 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T22:14:35,626 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T22:14:35,626 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T22:14:35,627 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T22:14:35,627 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T22:14:35,627 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T22:14:35,627 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T22:14:35,628 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T22:14:35,628 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T22:14:35,628 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T22:14:35,628 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T22:14:35,629 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T22:14:35,629 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T22:14:35,629 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T22:14:35,630 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T22:14:35,630 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T22:14:35,630 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T22:14:35,630 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T22:14:35,631 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T22:14:35,631 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T22:14:35,632 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T22:14:35,632 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T22:14:35,632 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T22:14:35,633 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T22:14:35,633 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T22:14:35,633 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T22:14:35,633 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T22:14:35,634 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T22:14:35,634 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T22:14:35,634 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T22:14:35,635 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T22:14:35,635 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T22:14:35,635 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T22:14:35,636 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T22:14:35,636 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T22:14:35,636 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T22:14:35,636 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T22:14:35,637 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T22:14:35,637 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T22:14:35,637 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T22:14:35,638 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T22:14:35,638 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T22:14:35,639 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T22:14:35,640 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T22:14:35,640 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T22:14:35,640 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T22:14:35,641 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T22:14:35,642 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T22:14:35,643 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T22:14:35,644 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T22:14:35,644 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T22:14:35,645 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T22:14:35,645 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T22:14:35,646 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T22:14:35,646 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T22:14:35,647 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T22:14:35,647 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T22:14:35,647 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T22:14:35,647 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T22:14:35,647 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T22:14:35,648 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T22:14:35,648 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T22:14:35,648 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T22:14:35,649 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T22:14:35,649 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T22:14:35,649 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T22:14:35,650 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T22:14:35,650 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T22:14:35,650 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T22:14:35,650 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T22:14:35,651 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T22:14:35,651 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T22:14:35,651 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T22:14:35,651 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T22:14:35,652 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T22:14:35,652 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T22:14:35,652 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T22:14:35,652 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T22:14:35,653 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T22:14:35,653 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T22:14:35,653 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T22:14:35,654 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T22:14:35,654 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T22:14:35,655 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T22:14:35,655 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T22:14:35,656 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T22:14:35,656 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T22:14:35,657 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T22:14:35,657 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T22:14:35,658 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T22:14:35,658 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T22:14:35,659 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T22:14:35,659 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T22:14:35,659 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T22:14:35,660 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T22:14:35,660 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T22:14:35,661 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T22:14:35,661 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T22:14:35,661 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T22:14:35,662 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T22:14:35,662 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T22:14:35,662 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T22:14:35,662 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T22:14:35,663 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T22:14:35,663 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T22:14:35,663 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T22:14:35,663 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T22:14:35,664 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T22:14:35,664 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T22:14:35,664 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T22:14:35,665 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T22:14:35,665 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T22:14:35,665 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T22:14:35,666 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T22:14:35,666 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T22:14:35,666 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T22:14:35,667 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T22:14:35,667 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T22:14:35,667 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T22:14:35,668 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T22:14:35,668 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T22:14:35,669 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T22:14:35,669 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T22:14:35,669 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T22:14:35,669 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T22:14:35,670 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T22:14:35,670 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T22:14:35,670 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T22:14:35,671 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T22:14:35,671 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T22:14:35,671 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T22:14:35,671 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T22:14:35,672 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T22:14:35,672 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T22:14:35,672 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T22:14:35,672 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T22:14:35,672 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T22:14:35,673 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T22:14:35,673 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T22:14:35,673 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T22:14:35,674 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T22:14:35,674 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T22:14:35,674 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T22:14:35,675 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T22:14:35,675 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T22:14:35,675 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T22:14:35,676 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T22:14:35,676 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T22:14:35,676 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T22:14:35,677 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T22:14:35,679 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T22:14:35,681 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T22:14:35,681 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T22:14:35,682 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T22:14:35,682 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T22:14:35,677 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T22:14:35,683 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T22:14:35,683 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T22:14:35,683 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T22:14:35,683 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T22:14:35,684 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T22:14:35,684 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T22:14:35,685 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T22:14:35,685 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T22:14:35,685 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T22:14:35,685 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T22:14:35,686 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T22:14:35,686 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T22:14:35,686 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T22:14:35,686 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T22:14:35,687 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T22:14:35,687 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T22:14:35,687 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T22:14:35,688 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T22:14:35,688 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T22:14:35,688 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T22:14:35,688 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T22:14:35,689 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T22:14:35,689 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T22:14:35,689 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T22:14:35,690 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T22:14:35,690 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T22:14:35,690 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T22:14:35,691 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T22:14:35,691 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T22:14:35,691 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T22:14:35,691 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T22:14:35,691 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T22:14:35,692 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T22:14:35,692 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T22:14:35,693 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T22:14:35,693 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T22:14:35,694 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T22:14:35,694 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T22:14:35,695 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T22:14:35,695 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T22:14:35,695 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T22:14:35,696 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T22:14:35,696 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T22:14:35,697 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T22:14:35,697 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T22:14:35,697 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T22:14:35,698 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T22:14:35,698 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T22:14:35,699 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T22:14:35,699 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T22:14:35,699 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T22:14:35,700 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T22:14:35,700 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T22:14:35,701 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T22:14:35,701 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T22:14:35,701 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T22:14:35,702 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T22:14:35,702 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T22:14:35,703 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T22:14:35,703 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T22:14:35,703 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T22:14:35,704 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T22:14:35,704 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T22:14:35,704 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T22:14:35,704 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T22:14:35,705 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T22:14:35,705 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T22:14:35,705 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T22:14:35,706 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T22:14:35,706 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T22:14:35,706 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T22:14:35,706 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T22:14:35,707 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T22:14:35,707 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T22:14:35,707 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T22:14:35,708 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T22:14:35,708 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T22:14:35,708 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T22:14:35,708 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T22:14:35,709 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T22:14:35,709 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T22:14:35,709 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T22:14:35,710 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T22:14:35,710 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T22:14:35,710 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T22:14:35,711 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T22:14:35,711 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T22:14:35,712 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T22:14:35,713 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T22:14:35,713 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T22:14:35,714 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T22:14:35,714 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T22:14:35,715 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T22:14:35,715 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T22:14:35,715 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T22:14:35,716 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T22:14:35,716 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T22:14:35,716 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T22:14:35,717 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T22:14:35,717 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T22:14:35,718 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T22:14:35,718 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T22:14:35,719 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T22:14:35,719 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T22:14:35,719 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T22:14:35,720 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T22:14:35,720 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T22:14:35,720 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T22:14:35,721 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T22:14:35,721 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T22:14:35,722 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T22:14:35,722 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T22:14:35,723 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T22:14:35,723 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T22:14:35,723 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T22:14:35,723 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T22:14:35,724 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T22:14:35,724 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T22:14:35,724 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T22:14:35,724 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T22:14:35,725 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T22:14:35,725 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T22:14:35,726 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T22:14:35,727 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T22:14:35,727 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T22:14:35,728 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T22:14:35,728 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T22:14:35,728 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T22:14:35,729 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T22:14:35,729 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T22:14:35,730 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T22:14:35,731 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T22:14:35,731 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T22:14:35,732 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T22:14:35,733 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T22:14:35,733 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T22:14:35,734 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T22:14:35,734 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T22:14:35,734 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T22:14:35,735 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T22:14:35,735 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T22:14:35,736 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T22:14:35,736 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T22:14:35,736 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T22:14:35,737 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T22:14:35,737 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T22:14:35,737 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T22:14:35,738 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T22:14:35,738 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T22:14:35,739 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T22:14:35,739 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T22:14:35,740 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T22:14:35,740 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T22:14:35,740 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T22:14:35,741 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T22:14:35,741 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T22:14:35,742 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T22:14:35,742 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T22:14:35,742 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T22:14:35,742 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T22:14:35,743 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T22:14:35,743 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T22:14:35,743 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T22:14:35,744 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T22:14:35,745 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T22:14:35,745 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T22:14:35,746 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T22:14:35,746 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T22:14:35,747 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T22:14:35,748 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T22:14:35,748 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T22:14:35,749 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T22:14:35,750 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T22:14:35,750 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T22:14:35,751 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T22:14:35,752 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T22:14:35,753 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T22:14:35,753 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T22:14:35,753 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T22:14:35,754 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T22:14:35,754 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T22:14:35,754 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T22:14:35,755 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T22:14:35,756 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T22:14:35,756 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T22:14:35,756 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T22:14:35,757 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T22:14:35,757 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T22:14:35,757 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T22:14:35,758 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T22:14:35,758 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T22:14:35,758 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T22:14:35,759 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T22:14:35,759 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T22:14:35,759 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T22:14:35,760 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T22:14:35,760 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T22:14:35,760 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T22:14:35,761 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T22:14:35,761 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T22:14:35,761 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T22:14:35,762 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T22:14:35,762 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T22:14:35,762 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T22:14:35,763 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T22:14:35,763 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T22:14:35,763 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T22:14:35,764 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T22:14:35,764 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T22:14:35,764 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T22:14:35,765 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T22:14:35,765 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T22:14:35,765 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T22:14:35,766 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T22:14:35,766 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T22:14:35,766 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T22:14:35,766 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T22:14:35,767 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T22:14:35,767 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T22:14:35,767 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T22:14:35,767 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T22:14:35,768 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T22:14:35,768 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T22:14:35,769 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T22:14:35,769 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T22:14:35,769 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T22:14:35,769 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T22:14:35,770 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T22:14:35,770 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T22:14:35,770 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T22:14:35,771 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T22:14:35,771 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T22:14:35,772 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T22:14:35,772 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T22:14:35,773 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T22:14:35,773 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T22:14:35,773 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T22:14:35,773 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T22:14:35,774 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T22:14:35,774 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T22:14:35,775 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T22:14:35,776 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T22:14:35,776 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T22:14:35,777 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T22:14:35,778 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T22:14:35,780 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T22:14:35,780 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T22:14:35,781 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T22:14:35,782 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T22:14:35,785 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T22:14:35,786 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T22:14:35,786 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T22:14:35,786 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40087 {}] regionserver.HRegion(8855): Flush requested on 63842f73863e9e7b9587486b122e6aae 2024-11-24T22:14:35,786 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T22:14:35,787 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 63842f73863e9e7b9587486b122e6aae 3/3 column families, dataSize=46.96 KB heapSize=140.20 KB 2024-11-24T22:14:35,787 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 63842f73863e9e7b9587486b122e6aae, store=A 2024-11-24T22:14:35,787 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T22:14:35,787 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-24T22:14:35,787 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 63842f73863e9e7b9587486b122e6aae, store=B 2024-11-24T22:14:35,787 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-24T22:14:35,787 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 63842f73863e9e7b9587486b122e6aae, store=C 2024-11-24T22:14:35,787 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-24T22:14:35,788 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T22:14:35,789 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T22:14:35,789 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T22:14:35,790 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T22:14:35,791 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T22:14:35,791 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T22:14:35,792 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T22:14:35,792 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T22:14:35,793 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T22:14:35,793 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T22:14:35,793 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T22:14:35,793 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T22:14:35,793 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T22:14:35,794 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T22:14:35,794 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T22:14:35,795 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T22:14:35,795 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T22:14:35,795 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T22:14:35,796 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T22:14:35,796 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T22:14:35,796 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T22:14:35,797 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T22:14:35,797 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T22:14:35,797 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T22:14:35,797 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T22:14:35,798 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T22:14:35,798 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T22:14:35,798 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T22:14:35,799 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T22:14:35,799 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T22:14:35,799 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T22:14:35,800 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T22:14:35,800 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T22:14:35,800 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T22:14:35,801 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T22:14:35,801 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T22:14:35,801 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T22:14:35,802 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T22:14:35,803 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T22:14:35,803 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T22:14:35,803 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T22:14:35,804 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T22:14:35,804 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T22:14:35,804 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T22:14:35,805 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T22:14:35,805 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T22:14:35,805 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T22:14:35,806 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241124e2c7467f191942dbb8139a28109001ad_63842f73863e9e7b9587486b122e6aae is 50, key is test_row_1/A:col10/1732486475784/Put/seqid=0 2024-11-24T22:14:35,806 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T22:14:35,806 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T22:14:35,806 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T22:14:35,807 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T22:14:35,807 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T22:14:35,808 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T22:14:35,808 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T22:14:35,808 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T22:14:35,809 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T22:14:35,809 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T22:14:35,810 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T22:14:35,810 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T22:14:35,813 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T22:14:35,814 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T22:14:35,814 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T22:14:35,815 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T22:14:35,816 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T22:14:35,817 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T22:14:35,817 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T22:14:35,818 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T22:14:35,819 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T22:14:35,820 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T22:14:35,821 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T22:14:35,822 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T22:14:35,822 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T22:14:35,824 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T22:14:35,825 DEBUG [RS:0;bd6592c37bb2:40087-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/63842f73863e9e7b9587486b122e6aae/.tmp/C/4923864c6fda45d6b701d9e800bd4c46 as hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/63842f73863e9e7b9587486b122e6aae/C/4923864c6fda45d6b701d9e800bd4c46 2024-11-24T22:14:35,825 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T22:14:35,826 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T22:14:35,826 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T22:14:35,827 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T22:14:35,827 DEBUG [RS:0;bd6592c37bb2:40087-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/63842f73863e9e7b9587486b122e6aae/.tmp/A/937c2f75b1e34c80aa8852008cd4dce4 as hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/63842f73863e9e7b9587486b122e6aae/A/937c2f75b1e34c80aa8852008cd4dce4 2024-11-24T22:14:35,828 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T22:14:35,828 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T22:14:35,829 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T22:14:35,830 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T22:14:35,830 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T22:14:35,830 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T22:14:35,832 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T22:14:35,832 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T22:14:35,833 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T22:14:35,833 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T22:14:35,833 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T22:14:35,835 INFO [RS:0;bd6592c37bb2:40087-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 63842f73863e9e7b9587486b122e6aae/C of 63842f73863e9e7b9587486b122e6aae into 4923864c6fda45d6b701d9e800bd4c46(size=12.3 K), total size for store is 24.2 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-24T22:14:35,835 DEBUG [RS:0;bd6592c37bb2:40087-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 63842f73863e9e7b9587486b122e6aae: 2024-11-24T22:14:35,835 INFO [RS:0;bd6592c37bb2:40087-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732486460633.63842f73863e9e7b9587486b122e6aae., storeName=63842f73863e9e7b9587486b122e6aae/C, priority=13, startTime=1732486474887; duration=0sec 2024-11-24T22:14:35,835 DEBUG [RS:0;bd6592c37bb2:40087-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-24T22:14:35,835 DEBUG [RS:0;bd6592c37bb2:40087-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 63842f73863e9e7b9587486b122e6aae:C 2024-11-24T22:14:35,838 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45929 is added to blk_1073741999_1175 (size=17284) 2024-11-24T22:14:35,839 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T22:14:35,843 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241124e2c7467f191942dbb8139a28109001ad_63842f73863e9e7b9587486b122e6aae to hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241124e2c7467f191942dbb8139a28109001ad_63842f73863e9e7b9587486b122e6aae 2024-11-24T22:14:35,845 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/63842f73863e9e7b9587486b122e6aae/.tmp/A/9563a22f4a1a47adbf57ca6ce108fb23, store: [table=TestAcidGuarantees family=A region=63842f73863e9e7b9587486b122e6aae] 2024-11-24T22:14:35,845 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/63842f73863e9e7b9587486b122e6aae/.tmp/A/9563a22f4a1a47adbf57ca6ce108fb23 is 175, key is test_row_1/A:col10/1732486475784/Put/seqid=0 2024-11-24T22:14:35,854 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45929 is added to blk_1073742000_1176 (size=48385) 2024-11-24T22:14:35,869 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T22:14:35,869 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T22:14:35,870 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T22:14:35,870 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T22:14:35,870 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T22:14:35,870 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T22:14:35,871 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T22:14:35,871 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T22:14:35,871 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T22:14:35,872 INFO [RS:0;bd6592c37bb2:40087-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 63842f73863e9e7b9587486b122e6aae/A of 63842f73863e9e7b9587486b122e6aae into 937c2f75b1e34c80aa8852008cd4dce4(size=30.8 K), total size for store is 61.2 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-24T22:14:35,872 DEBUG [RS:0;bd6592c37bb2:40087-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 63842f73863e9e7b9587486b122e6aae: 2024-11-24T22:14:35,872 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T22:14:35,872 INFO [RS:0;bd6592c37bb2:40087-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732486460633.63842f73863e9e7b9587486b122e6aae., storeName=63842f73863e9e7b9587486b122e6aae/A, priority=13, startTime=1732486474887; duration=0sec 2024-11-24T22:14:35,872 DEBUG [RS:0;bd6592c37bb2:40087-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-24T22:14:35,872 DEBUG [RS:0;bd6592c37bb2:40087-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 63842f73863e9e7b9587486b122e6aae:A 2024-11-24T22:14:35,872 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T22:14:35,872 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T22:14:35,873 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T22:14:35,873 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T22:14:35,874 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T22:14:35,874 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T22:14:35,874 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T22:14:35,875 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T22:14:35,875 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T22:14:35,875 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T22:14:35,876 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T22:14:35,876 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T22:14:35,876 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T22:14:35,877 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T22:14:35,877 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T22:14:35,878 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T22:14:35,878 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T22:14:35,878 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T22:14:35,879 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T22:14:35,879 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T22:14:35,880 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T22:14:35,880 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T22:14:35,880 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T22:14:35,881 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T22:14:35,881 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T22:14:35,881 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T22:14:35,881 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T22:14:35,881 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T22:14:35,882 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T22:14:35,882 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T22:14:35,883 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T22:14:35,883 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T22:14:35,884 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T22:14:35,884 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T22:14:35,884 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T22:14:35,885 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T22:14:35,887 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T22:14:35,887 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T22:14:35,888 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T22:14:35,888 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T22:14:35,888 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T22:14:35,889 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T22:14:35,889 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T22:14:35,890 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T22:14:35,890 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T22:14:35,890 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T22:14:35,891 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T22:14:35,892 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T22:14:35,892 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T22:14:35,900 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T22:14:35,901 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T22:14:35,902 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T22:14:35,902 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T22:14:35,903 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T22:14:35,904 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T22:14:35,904 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T22:14:35,905 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T22:14:35,905 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T22:14:35,906 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T22:14:35,906 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T22:14:35,906 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T22:14:35,907 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T22:14:35,907 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T22:14:35,908 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T22:14:35,908 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T22:14:35,909 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T22:14:35,913 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T22:14:35,913 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T22:14:35,913 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T22:14:35,914 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T22:14:35,914 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T22:14:35,914 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T22:14:35,914 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T22:14:35,915 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T22:14:35,915 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T22:14:35,916 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T22:14:35,916 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T22:14:35,916 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T22:14:35,916 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T22:14:35,917 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T22:14:35,917 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T22:14:35,918 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T22:14:35,918 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T22:14:35,919 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T22:14:35,920 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T22:14:35,920 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T22:14:35,920 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T22:14:35,920 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T22:14:35,921 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T22:14:35,930 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T22:14:35,932 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T22:14:35,933 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T22:14:35,941 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T22:14:35,941 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T22:14:35,942 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T22:14:35,943 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T22:14:35,944 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T22:14:35,944 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T22:14:35,945 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T22:14:35,946 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T22:14:35,946 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T22:14:35,949 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T22:14:35,949 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T22:14:35,950 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T22:14:35,951 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T22:14:35,952 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T22:14:35,952 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T22:14:35,953 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T22:14:35,954 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T22:14:35,954 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T22:14:35,955 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T22:14:35,956 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T22:14:35,956 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T22:14:35,957 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T22:14:35,957 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T22:14:35,958 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T22:14:35,959 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T22:14:35,959 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T22:14:35,973 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/bd6592c37bb2:0-5 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=309, cellSize=14145, totalHeapSize=32376, min timestamp=1732486475931, max timestamp=1732486475969Num uniques 859; 2024-11-24T22:14:35,973 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/bd6592c37bb2:0-5 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=14145, suffix heap size=117064,new segment heap size=32376 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=309 2024-11-24T22:14:35,975 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/bd6592c37bb2:0-7 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=309, cellSize=14145, totalHeapSize=32376, min timestamp=1732486475931, max timestamp=1732486475969Num uniques 859; 2024-11-24T22:14:35,975 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/bd6592c37bb2:0-7 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=14145, suffix heap size=117064,new segment heap size=32376 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=309 2024-11-24T22:14:35,976 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/bd6592c37bb2:0-5 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=309, cellSize=14145, totalHeapSize=32376, min timestamp=1732486475931, max timestamp=1732486475969Num uniques 859; 2024-11-24T22:14:35,976 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/bd6592c37bb2:0-5 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=14145, suffix heap size=117064,new segment heap size=32376 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=309 2024-11-24T22:14:36,034 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40087 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=63842f73863e9e7b9587486b122e6aae, server=bd6592c37bb2,40087,1732486423713 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-24T22:14:36,034 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40087 {}] ipc.CallRunner(138): callId: 106 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40206 deadline: 1732486536030, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=63842f73863e9e7b9587486b122e6aae, server=bd6592c37bb2,40087,1732486423713 2024-11-24T22:14:36,034 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40087 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=63842f73863e9e7b9587486b122e6aae, server=bd6592c37bb2,40087,1732486423713 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-24T22:14:36,034 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732486460633.63842f73863e9e7b9587486b122e6aae., hostname=bd6592c37bb2,40087,1732486423713, seqNum=5 , the old value is region=TestAcidGuarantees,,1732486460633.63842f73863e9e7b9587486b122e6aae., hostname=bd6592c37bb2,40087,1732486423713, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=63842f73863e9e7b9587486b122e6aae, server=bd6592c37bb2,40087,1732486423713 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T22:14:36,034 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732486460633.63842f73863e9e7b9587486b122e6aae., hostname=bd6592c37bb2,40087,1732486423713, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=63842f73863e9e7b9587486b122e6aae, server=bd6592c37bb2,40087,1732486423713 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T22:14:36,034 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40087 {}] ipc.CallRunner(138): callId: 98 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40244 deadline: 1732486536034, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=63842f73863e9e7b9587486b122e6aae, server=bd6592c37bb2,40087,1732486423713 2024-11-24T22:14:36,034 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732486460633.63842f73863e9e7b9587486b122e6aae., hostname=bd6592c37bb2,40087,1732486423713, seqNum=5 because the exception is null or not the one we care about 2024-11-24T22:14:36,035 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732486460633.63842f73863e9e7b9587486b122e6aae., hostname=bd6592c37bb2,40087,1732486423713, seqNum=5 , the old value is region=TestAcidGuarantees,,1732486460633.63842f73863e9e7b9587486b122e6aae., hostname=bd6592c37bb2,40087,1732486423713, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=63842f73863e9e7b9587486b122e6aae, server=bd6592c37bb2,40087,1732486423713 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T22:14:36,035 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732486460633.63842f73863e9e7b9587486b122e6aae., hostname=bd6592c37bb2,40087,1732486423713, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=63842f73863e9e7b9587486b122e6aae, server=bd6592c37bb2,40087,1732486423713 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T22:14:36,035 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732486460633.63842f73863e9e7b9587486b122e6aae., hostname=bd6592c37bb2,40087,1732486423713, seqNum=5 because the exception is null or not the one we care about 2024-11-24T22:14:36,149 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40087 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=63842f73863e9e7b9587486b122e6aae, server=bd6592c37bb2,40087,1732486423713 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-24T22:14:36,149 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40087 {}] ipc.CallRunner(138): callId: 107 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40206 deadline: 1732486536141, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=63842f73863e9e7b9587486b122e6aae, server=bd6592c37bb2,40087,1732486423713 2024-11-24T22:14:36,150 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40087 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=63842f73863e9e7b9587486b122e6aae, server=bd6592c37bb2,40087,1732486423713 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-24T22:14:36,150 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732486460633.63842f73863e9e7b9587486b122e6aae., hostname=bd6592c37bb2,40087,1732486423713, seqNum=5 , the old value is region=TestAcidGuarantees,,1732486460633.63842f73863e9e7b9587486b122e6aae., hostname=bd6592c37bb2,40087,1732486423713, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=63842f73863e9e7b9587486b122e6aae, server=bd6592c37bb2,40087,1732486423713 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T22:14:36,150 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732486460633.63842f73863e9e7b9587486b122e6aae., hostname=bd6592c37bb2,40087,1732486423713, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=63842f73863e9e7b9587486b122e6aae, server=bd6592c37bb2,40087,1732486423713 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T22:14:36,150 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40087 {}] ipc.CallRunner(138): callId: 99 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40244 deadline: 1732486536141, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=63842f73863e9e7b9587486b122e6aae, server=bd6592c37bb2,40087,1732486423713 2024-11-24T22:14:36,150 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732486460633.63842f73863e9e7b9587486b122e6aae., hostname=bd6592c37bb2,40087,1732486423713, seqNum=5 because the exception is null or not the one we care about 2024-11-24T22:14:36,150 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732486460633.63842f73863e9e7b9587486b122e6aae., hostname=bd6592c37bb2,40087,1732486423713, seqNum=5 , the old value is region=TestAcidGuarantees,,1732486460633.63842f73863e9e7b9587486b122e6aae., hostname=bd6592c37bb2,40087,1732486423713, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=63842f73863e9e7b9587486b122e6aae, server=bd6592c37bb2,40087,1732486423713 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T22:14:36,150 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732486460633.63842f73863e9e7b9587486b122e6aae., hostname=bd6592c37bb2,40087,1732486423713, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=63842f73863e9e7b9587486b122e6aae, server=bd6592c37bb2,40087,1732486423713 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T22:14:36,150 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732486460633.63842f73863e9e7b9587486b122e6aae., hostname=bd6592c37bb2,40087,1732486423713, seqNum=5 because the exception is null or not the one we care about 2024-11-24T22:14:36,256 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=220, memsize=15.7 K, hasBloomFilter=true, into tmp file hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/63842f73863e9e7b9587486b122e6aae/.tmp/A/9563a22f4a1a47adbf57ca6ce108fb23 2024-11-24T22:14:36,268 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/63842f73863e9e7b9587486b122e6aae/.tmp/B/fd1b77c22a4c4a79a625caa3d88bbeb4 is 50, key is test_row_1/B:col10/1732486475784/Put/seqid=0 2024-11-24T22:14:36,289 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45929 is added to blk_1073742001_1177 (size=9757) 2024-11-24T22:14:36,294 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40087 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=63842f73863e9e7b9587486b122e6aae, server=bd6592c37bb2,40087,1732486423713 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-24T22:14:36,294 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40087 {}] ipc.CallRunner(138): callId: 38 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40232 deadline: 1732486536290, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=63842f73863e9e7b9587486b122e6aae, server=bd6592c37bb2,40087,1732486423713 2024-11-24T22:14:36,294 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40087 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=63842f73863e9e7b9587486b122e6aae, server=bd6592c37bb2,40087,1732486423713 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-24T22:14:36,294 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40087 {}] ipc.CallRunner(138): callId: 38 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40226 deadline: 1732486536291, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=63842f73863e9e7b9587486b122e6aae, server=bd6592c37bb2,40087,1732486423713 2024-11-24T22:14:36,297 WARN [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRpcRetryingCaller(177): Call to bd6592c37bb2,40087,1732486423713 for 'test_row_2' in 63842f73863e9e7b9587486b122e6aae of TestAcidGuarantees failed, tries = 8, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 8208 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=63842f73863e9e7b9587486b122e6aae, server=bd6592c37bb2,40087,1732486423713 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor44.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=63842f73863e9e7b9587486b122e6aae, server=bd6592c37bb2,40087,1732486423713 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-24T22:14:36,296 WARN [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRpcRetryingCaller(177): Call to bd6592c37bb2,40087,1732486423713 for 'test_row_1' in 63842f73863e9e7b9587486b122e6aae of TestAcidGuarantees failed, tries = 8, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 8201 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=63842f73863e9e7b9587486b122e6aae, server=bd6592c37bb2,40087,1732486423713 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor44.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=63842f73863e9e7b9587486b122e6aae, server=bd6592c37bb2,40087,1732486423713 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-24T22:14:36,297 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732486460633.63842f73863e9e7b9587486b122e6aae., hostname=bd6592c37bb2,40087,1732486423713, seqNum=5 , the old value is region=TestAcidGuarantees,,1732486460633.63842f73863e9e7b9587486b122e6aae., hostname=bd6592c37bb2,40087,1732486423713, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=63842f73863e9e7b9587486b122e6aae, server=bd6592c37bb2,40087,1732486423713 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T22:14:36,297 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732486460633.63842f73863e9e7b9587486b122e6aae., hostname=bd6592c37bb2,40087,1732486423713, seqNum=5 , the old value is region=TestAcidGuarantees,,1732486460633.63842f73863e9e7b9587486b122e6aae., hostname=bd6592c37bb2,40087,1732486423713, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=63842f73863e9e7b9587486b122e6aae, server=bd6592c37bb2,40087,1732486423713 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T22:14:36,298 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732486460633.63842f73863e9e7b9587486b122e6aae., hostname=bd6592c37bb2,40087,1732486423713, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=63842f73863e9e7b9587486b122e6aae, server=bd6592c37bb2,40087,1732486423713 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T22:14:36,298 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732486460633.63842f73863e9e7b9587486b122e6aae., hostname=bd6592c37bb2,40087,1732486423713, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=63842f73863e9e7b9587486b122e6aae, server=bd6592c37bb2,40087,1732486423713 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T22:14:36,298 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732486460633.63842f73863e9e7b9587486b122e6aae., hostname=bd6592c37bb2,40087,1732486423713, seqNum=5 because the exception is null or not the one we care about 2024-11-24T22:14:36,298 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732486460633.63842f73863e9e7b9587486b122e6aae., hostname=bd6592c37bb2,40087,1732486423713, seqNum=5 because the exception is null or not the one we care about 2024-11-24T22:14:36,361 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40087 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=63842f73863e9e7b9587486b122e6aae, server=bd6592c37bb2,40087,1732486423713 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-24T22:14:36,361 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40087 {}] ipc.CallRunner(138): callId: 108 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40206 deadline: 1732486536361, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=63842f73863e9e7b9587486b122e6aae, server=bd6592c37bb2,40087,1732486423713 2024-11-24T22:14:36,362 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40087 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=63842f73863e9e7b9587486b122e6aae, server=bd6592c37bb2,40087,1732486423713 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-24T22:14:36,362 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732486460633.63842f73863e9e7b9587486b122e6aae., hostname=bd6592c37bb2,40087,1732486423713, seqNum=5 , the old value is region=TestAcidGuarantees,,1732486460633.63842f73863e9e7b9587486b122e6aae., hostname=bd6592c37bb2,40087,1732486423713, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=63842f73863e9e7b9587486b122e6aae, server=bd6592c37bb2,40087,1732486423713 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T22:14:36,362 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732486460633.63842f73863e9e7b9587486b122e6aae., hostname=bd6592c37bb2,40087,1732486423713, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=63842f73863e9e7b9587486b122e6aae, server=bd6592c37bb2,40087,1732486423713 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T22:14:36,362 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40087 {}] ipc.CallRunner(138): callId: 100 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40244 deadline: 1732486536361, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=63842f73863e9e7b9587486b122e6aae, server=bd6592c37bb2,40087,1732486423713 2024-11-24T22:14:36,362 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732486460633.63842f73863e9e7b9587486b122e6aae., hostname=bd6592c37bb2,40087,1732486423713, seqNum=5 because the exception is null or not the one we care about 2024-11-24T22:14:36,362 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732486460633.63842f73863e9e7b9587486b122e6aae., hostname=bd6592c37bb2,40087,1732486423713, seqNum=5 , the old value is region=TestAcidGuarantees,,1732486460633.63842f73863e9e7b9587486b122e6aae., hostname=bd6592c37bb2,40087,1732486423713, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=63842f73863e9e7b9587486b122e6aae, server=bd6592c37bb2,40087,1732486423713 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T22:14:36,362 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732486460633.63842f73863e9e7b9587486b122e6aae., hostname=bd6592c37bb2,40087,1732486423713, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=63842f73863e9e7b9587486b122e6aae, server=bd6592c37bb2,40087,1732486423713 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T22:14:36,362 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732486460633.63842f73863e9e7b9587486b122e6aae., hostname=bd6592c37bb2,40087,1732486423713, seqNum=5 because the exception is null or not the one we care about 2024-11-24T22:14:36,452 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37701 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=44 2024-11-24T22:14:36,453 INFO [RPCClient-NioEventLoopGroup-4-7 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-24T22:14:36,453 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37701 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-24T22:14:36,454 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37701 {}] procedure2.ProcedureExecutor(1139): Stored pid=46, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=46, table=TestAcidGuarantees 2024-11-24T22:14:36,455 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37701 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=46 2024-11-24T22:14:36,456 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=46, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=46, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-24T22:14:36,457 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=46, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=46, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-24T22:14:36,457 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=47, ppid=46, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-24T22:14:36,561 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37701 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=46 2024-11-24T22:14:36,611 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=40087 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=47 2024-11-24T22:14:36,611 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bd6592c37bb2:0-0 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732486460633.63842f73863e9e7b9587486b122e6aae. 2024-11-24T22:14:36,611 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bd6592c37bb2:0-0 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732486460633.63842f73863e9e7b9587486b122e6aae. as already flushing 2024-11-24T22:14:36,611 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bd6592c37bb2:0-0 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732486460633.63842f73863e9e7b9587486b122e6aae. 2024-11-24T22:14:36,611 ERROR [RS_FLUSH_OPERATIONS-regionserver/bd6592c37bb2:0-0 {event_type=RS_FLUSH_REGIONS, pid=47}] handler.RSProcedureHandler(58): pid=47 java.io.IOException: Unable to complete flush {ENCODED => 63842f73863e9e7b9587486b122e6aae, NAME => 'TestAcidGuarantees,,1732486460633.63842f73863e9e7b9587486b122e6aae.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T22:14:36,611 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bd6592c37bb2:0-0 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=47 java.io.IOException: Unable to complete flush {ENCODED => 63842f73863e9e7b9587486b122e6aae, NAME => 'TestAcidGuarantees,,1732486460633.63842f73863e9e7b9587486b122e6aae.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T22:14:36,612 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37701 {}] master.HMaster(4177): Remote procedure failed, pid=47 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 63842f73863e9e7b9587486b122e6aae, NAME => 'TestAcidGuarantees,,1732486460633.63842f73863e9e7b9587486b122e6aae.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 63842f73863e9e7b9587486b122e6aae, NAME => 'TestAcidGuarantees,,1732486460633.63842f73863e9e7b9587486b122e6aae.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T22:14:36,671 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40087 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=63842f73863e9e7b9587486b122e6aae, server=bd6592c37bb2,40087,1732486423713 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-24T22:14:36,671 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40087 {}] ipc.CallRunner(138): callId: 109 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40206 deadline: 1732486536671, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=63842f73863e9e7b9587486b122e6aae, server=bd6592c37bb2,40087,1732486423713 2024-11-24T22:14:36,672 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40087 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=63842f73863e9e7b9587486b122e6aae, server=bd6592c37bb2,40087,1732486423713 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-24T22:14:36,672 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732486460633.63842f73863e9e7b9587486b122e6aae., hostname=bd6592c37bb2,40087,1732486423713, seqNum=5 , the old value is region=TestAcidGuarantees,,1732486460633.63842f73863e9e7b9587486b122e6aae., hostname=bd6592c37bb2,40087,1732486423713, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=63842f73863e9e7b9587486b122e6aae, server=bd6592c37bb2,40087,1732486423713 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T22:14:36,672 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732486460633.63842f73863e9e7b9587486b122e6aae., hostname=bd6592c37bb2,40087,1732486423713, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=63842f73863e9e7b9587486b122e6aae, server=bd6592c37bb2,40087,1732486423713 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T22:14:36,672 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40087 {}] ipc.CallRunner(138): callId: 101 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40244 deadline: 1732486536671, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=63842f73863e9e7b9587486b122e6aae, server=bd6592c37bb2,40087,1732486423713 2024-11-24T22:14:36,672 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732486460633.63842f73863e9e7b9587486b122e6aae., hostname=bd6592c37bb2,40087,1732486423713, seqNum=5 because the exception is null or not the one we care about 2024-11-24T22:14:36,672 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732486460633.63842f73863e9e7b9587486b122e6aae., hostname=bd6592c37bb2,40087,1732486423713, seqNum=5 , the old value is region=TestAcidGuarantees,,1732486460633.63842f73863e9e7b9587486b122e6aae., hostname=bd6592c37bb2,40087,1732486423713, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=63842f73863e9e7b9587486b122e6aae, server=bd6592c37bb2,40087,1732486423713 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T22:14:36,672 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732486460633.63842f73863e9e7b9587486b122e6aae., hostname=bd6592c37bb2,40087,1732486423713, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=63842f73863e9e7b9587486b122e6aae, server=bd6592c37bb2,40087,1732486423713 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T22:14:36,672 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732486460633.63842f73863e9e7b9587486b122e6aae., hostname=bd6592c37bb2,40087,1732486423713, seqNum=5 because the exception is null or not the one we care about 2024-11-24T22:14:36,690 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=220 (bloomFilter=true), to=hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/63842f73863e9e7b9587486b122e6aae/.tmp/B/fd1b77c22a4c4a79a625caa3d88bbeb4 2024-11-24T22:14:36,699 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/63842f73863e9e7b9587486b122e6aae/.tmp/C/f9001c2421d641e1a3a5fa501a7bf407 is 50, key is test_row_1/C:col10/1732486475784/Put/seqid=0 2024-11-24T22:14:36,717 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45929 is added to blk_1073742002_1178 (size=9757) 2024-11-24T22:14:36,769 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=40087 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=47 2024-11-24T22:14:36,769 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bd6592c37bb2:0-1 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732486460633.63842f73863e9e7b9587486b122e6aae. 2024-11-24T22:14:36,770 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bd6592c37bb2:0-1 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732486460633.63842f73863e9e7b9587486b122e6aae. as already flushing 2024-11-24T22:14:36,770 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bd6592c37bb2:0-1 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732486460633.63842f73863e9e7b9587486b122e6aae. 2024-11-24T22:14:36,770 ERROR [RS_FLUSH_OPERATIONS-regionserver/bd6592c37bb2:0-1 {event_type=RS_FLUSH_REGIONS, pid=47}] handler.RSProcedureHandler(58): pid=47 java.io.IOException: Unable to complete flush {ENCODED => 63842f73863e9e7b9587486b122e6aae, NAME => 'TestAcidGuarantees,,1732486460633.63842f73863e9e7b9587486b122e6aae.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T22:14:36,771 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bd6592c37bb2:0-1 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=47 java.io.IOException: Unable to complete flush {ENCODED => 63842f73863e9e7b9587486b122e6aae, NAME => 'TestAcidGuarantees,,1732486460633.63842f73863e9e7b9587486b122e6aae.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T22:14:36,771 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37701 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=46 2024-11-24T22:14:36,772 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37701 {}] master.HMaster(4177): Remote procedure failed, pid=47 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 63842f73863e9e7b9587486b122e6aae, NAME => 'TestAcidGuarantees,,1732486460633.63842f73863e9e7b9587486b122e6aae.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 63842f73863e9e7b9587486b122e6aae, NAME => 'TestAcidGuarantees,,1732486460633.63842f73863e9e7b9587486b122e6aae.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T22:14:36,924 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=40087 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=47 2024-11-24T22:14:36,924 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bd6592c37bb2:0-2 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732486460633.63842f73863e9e7b9587486b122e6aae. 2024-11-24T22:14:36,924 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bd6592c37bb2:0-2 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732486460633.63842f73863e9e7b9587486b122e6aae. as already flushing 2024-11-24T22:14:36,924 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bd6592c37bb2:0-2 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732486460633.63842f73863e9e7b9587486b122e6aae. 2024-11-24T22:14:36,924 ERROR [RS_FLUSH_OPERATIONS-regionserver/bd6592c37bb2:0-2 {event_type=RS_FLUSH_REGIONS, pid=47}] handler.RSProcedureHandler(58): pid=47 java.io.IOException: Unable to complete flush {ENCODED => 63842f73863e9e7b9587486b122e6aae, NAME => 'TestAcidGuarantees,,1732486460633.63842f73863e9e7b9587486b122e6aae.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T22:14:36,925 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bd6592c37bb2:0-2 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=47 java.io.IOException: Unable to complete flush {ENCODED => 63842f73863e9e7b9587486b122e6aae, NAME => 'TestAcidGuarantees,,1732486460633.63842f73863e9e7b9587486b122e6aae.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T22:14:36,925 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37701 {}] master.HMaster(4177): Remote procedure failed, pid=47 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 63842f73863e9e7b9587486b122e6aae, NAME => 'TestAcidGuarantees,,1732486460633.63842f73863e9e7b9587486b122e6aae.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 63842f73863e9e7b9587486b122e6aae, NAME => 'TestAcidGuarantees,,1732486460633.63842f73863e9e7b9587486b122e6aae.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T22:14:37,077 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=40087 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=47 2024-11-24T22:14:37,077 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bd6592c37bb2:0-0 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732486460633.63842f73863e9e7b9587486b122e6aae. 2024-11-24T22:14:37,078 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bd6592c37bb2:0-0 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732486460633.63842f73863e9e7b9587486b122e6aae. as already flushing 2024-11-24T22:14:37,078 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bd6592c37bb2:0-0 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732486460633.63842f73863e9e7b9587486b122e6aae. 2024-11-24T22:14:37,078 ERROR [RS_FLUSH_OPERATIONS-regionserver/bd6592c37bb2:0-0 {event_type=RS_FLUSH_REGIONS, pid=47}] handler.RSProcedureHandler(58): pid=47 java.io.IOException: Unable to complete flush {ENCODED => 63842f73863e9e7b9587486b122e6aae, NAME => 'TestAcidGuarantees,,1732486460633.63842f73863e9e7b9587486b122e6aae.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T22:14:37,078 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bd6592c37bb2:0-0 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=47 java.io.IOException: Unable to complete flush {ENCODED => 63842f73863e9e7b9587486b122e6aae, NAME => 'TestAcidGuarantees,,1732486460633.63842f73863e9e7b9587486b122e6aae.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T22:14:37,079 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37701 {}] master.HMaster(4177): Remote procedure failed, pid=47 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 63842f73863e9e7b9587486b122e6aae, NAME => 'TestAcidGuarantees,,1732486460633.63842f73863e9e7b9587486b122e6aae.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 63842f73863e9e7b9587486b122e6aae, NAME => 'TestAcidGuarantees,,1732486460633.63842f73863e9e7b9587486b122e6aae.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T22:14:37,081 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37701 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=46 2024-11-24T22:14:37,118 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=220 (bloomFilter=true), to=hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/63842f73863e9e7b9587486b122e6aae/.tmp/C/f9001c2421d641e1a3a5fa501a7bf407 2024-11-24T22:14:37,124 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/63842f73863e9e7b9587486b122e6aae/.tmp/A/9563a22f4a1a47adbf57ca6ce108fb23 as hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/63842f73863e9e7b9587486b122e6aae/A/9563a22f4a1a47adbf57ca6ce108fb23 2024-11-24T22:14:37,129 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/63842f73863e9e7b9587486b122e6aae/A/9563a22f4a1a47adbf57ca6ce108fb23, entries=250, sequenceid=220, filesize=47.3 K 2024-11-24T22:14:37,130 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/63842f73863e9e7b9587486b122e6aae/.tmp/B/fd1b77c22a4c4a79a625caa3d88bbeb4 as hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/63842f73863e9e7b9587486b122e6aae/B/fd1b77c22a4c4a79a625caa3d88bbeb4 2024-11-24T22:14:37,135 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/63842f73863e9e7b9587486b122e6aae/B/fd1b77c22a4c4a79a625caa3d88bbeb4, entries=100, sequenceid=220, filesize=9.5 K 2024-11-24T22:14:37,138 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/63842f73863e9e7b9587486b122e6aae/.tmp/C/f9001c2421d641e1a3a5fa501a7bf407 as hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/63842f73863e9e7b9587486b122e6aae/C/f9001c2421d641e1a3a5fa501a7bf407 2024-11-24T22:14:37,145 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/63842f73863e9e7b9587486b122e6aae/C/f9001c2421d641e1a3a5fa501a7bf407, entries=100, sequenceid=220, filesize=9.5 K 2024-11-24T22:14:37,150 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~46.96 KB/48090, heapSize ~140.16 KB/143520, currentSize=140.89 KB/144270 for 63842f73863e9e7b9587486b122e6aae in 1362ms, sequenceid=220, compaction requested=true 2024-11-24T22:14:37,150 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 63842f73863e9e7b9587486b122e6aae: 2024-11-24T22:14:37,150 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 63842f73863e9e7b9587486b122e6aae:A, priority=-2147483648, current under compaction store size is 1 2024-11-24T22:14:37,150 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-24T22:14:37,150 DEBUG [RS:0;bd6592c37bb2:40087-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-24T22:14:37,150 DEBUG [RS:0;bd6592c37bb2:40087-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-24T22:14:37,150 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 63842f73863e9e7b9587486b122e6aae:B, priority=-2147483648, current under compaction store size is 2 2024-11-24T22:14:37,150 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-24T22:14:37,150 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 63842f73863e9e7b9587486b122e6aae:C, priority=-2147483648, current under compaction store size is 3 2024-11-24T22:14:37,150 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-24T22:14:37,151 DEBUG [RS:0;bd6592c37bb2:40087-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 111039 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-24T22:14:37,151 DEBUG [RS:0;bd6592c37bb2:40087-shortCompactions-0 {}] regionserver.HStore(1541): 63842f73863e9e7b9587486b122e6aae/A is initiating minor compaction (all files) 2024-11-24T22:14:37,152 INFO [RS:0;bd6592c37bb2:40087-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 63842f73863e9e7b9587486b122e6aae/A in TestAcidGuarantees,,1732486460633.63842f73863e9e7b9587486b122e6aae. 2024-11-24T22:14:37,152 INFO [RS:0;bd6592c37bb2:40087-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/63842f73863e9e7b9587486b122e6aae/A/937c2f75b1e34c80aa8852008cd4dce4, hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/63842f73863e9e7b9587486b122e6aae/A/a80b645125754d9299915c8d52dab903, hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/63842f73863e9e7b9587486b122e6aae/A/9563a22f4a1a47adbf57ca6ce108fb23] into tmpdir=hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/63842f73863e9e7b9587486b122e6aae/.tmp, totalSize=108.4 K 2024-11-24T22:14:37,152 INFO [RS:0;bd6592c37bb2:40087-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=13 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1732486460633.63842f73863e9e7b9587486b122e6aae. 2024-11-24T22:14:37,152 DEBUG [RS:0;bd6592c37bb2:40087-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1732486460633.63842f73863e9e7b9587486b122e6aae. files: [hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/63842f73863e9e7b9587486b122e6aae/A/937c2f75b1e34c80aa8852008cd4dce4, hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/63842f73863e9e7b9587486b122e6aae/A/a80b645125754d9299915c8d52dab903, hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/63842f73863e9e7b9587486b122e6aae/A/9563a22f4a1a47adbf57ca6ce108fb23] 2024-11-24T22:14:37,152 DEBUG [RS:0;bd6592c37bb2:40087-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 34503 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-24T22:14:37,152 DEBUG [RS:0;bd6592c37bb2:40087-longCompactions-0 {}] regionserver.HStore(1541): 63842f73863e9e7b9587486b122e6aae/B is initiating minor compaction (all files) 2024-11-24T22:14:37,152 INFO [RS:0;bd6592c37bb2:40087-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 63842f73863e9e7b9587486b122e6aae/B in TestAcidGuarantees,,1732486460633.63842f73863e9e7b9587486b122e6aae. 2024-11-24T22:14:37,152 INFO [RS:0;bd6592c37bb2:40087-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/63842f73863e9e7b9587486b122e6aae/B/7a02ad10845946b78864d3662caaedbd, hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/63842f73863e9e7b9587486b122e6aae/B/80c3be87a7bc406abcd49b8ff894561f, hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/63842f73863e9e7b9587486b122e6aae/B/fd1b77c22a4c4a79a625caa3d88bbeb4] into tmpdir=hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/63842f73863e9e7b9587486b122e6aae/.tmp, totalSize=33.7 K 2024-11-24T22:14:37,153 DEBUG [RS:0;bd6592c37bb2:40087-shortCompactions-0 {}] compactions.Compactor(225): Compacting 937c2f75b1e34c80aa8852008cd4dce4, keycount=150, bloomtype=ROW, size=30.8 K, encoding=NONE, compression=NONE, seqNum=194, earliestPutTs=1732486471251 2024-11-24T22:14:37,153 DEBUG [RS:0;bd6592c37bb2:40087-shortCompactions-0 {}] compactions.Compactor(225): Compacting a80b645125754d9299915c8d52dab903, keycount=150, bloomtype=ROW, size=30.4 K, encoding=NONE, compression=NONE, seqNum=209, earliestPutTs=1732486473447 2024-11-24T22:14:37,154 DEBUG [RS:0;bd6592c37bb2:40087-longCompactions-0 {}] compactions.Compactor(225): Compacting 7a02ad10845946b78864d3662caaedbd, keycount=150, bloomtype=ROW, size=12.3 K, encoding=NONE, compression=NONE, seqNum=194, earliestPutTs=1732486471251 2024-11-24T22:14:37,154 DEBUG [RS:0;bd6592c37bb2:40087-shortCompactions-0 {}] compactions.Compactor(225): Compacting 9563a22f4a1a47adbf57ca6ce108fb23, keycount=250, bloomtype=ROW, size=47.3 K, encoding=NONE, compression=NONE, seqNum=220, earliestPutTs=1732486475713 2024-11-24T22:14:37,155 DEBUG [RS:0;bd6592c37bb2:40087-longCompactions-0 {}] compactions.Compactor(225): Compacting 80c3be87a7bc406abcd49b8ff894561f, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=209, earliestPutTs=1732486473447 2024-11-24T22:14:37,155 DEBUG [RS:0;bd6592c37bb2:40087-longCompactions-0 {}] compactions.Compactor(225): Compacting fd1b77c22a4c4a79a625caa3d88bbeb4, keycount=100, bloomtype=ROW, size=9.5 K, encoding=NONE, compression=NONE, seqNum=220, earliestPutTs=1732486475776 2024-11-24T22:14:37,170 INFO [RS:0;bd6592c37bb2:40087-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=63842f73863e9e7b9587486b122e6aae] 2024-11-24T22:14:37,179 INFO [RS:0;bd6592c37bb2:40087-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 63842f73863e9e7b9587486b122e6aae#B#compaction#148 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-24T22:14:37,180 DEBUG [RS:0;bd6592c37bb2:40087-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/63842f73863e9e7b9587486b122e6aae/.tmp/B/455dcb4f2aca48b98940fe2ebddff3fb is 50, key is test_row_0/B:col10/1732486473479/Put/seqid=0 2024-11-24T22:14:37,185 DEBUG [RS:0;bd6592c37bb2:40087-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e202411246cb35d08b94b451ca806407561820980_63842f73863e9e7b9587486b122e6aae store=[table=TestAcidGuarantees family=A region=63842f73863e9e7b9587486b122e6aae] 2024-11-24T22:14:37,187 DEBUG [RS:0;bd6592c37bb2:40087-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e202411246cb35d08b94b451ca806407561820980_63842f73863e9e7b9587486b122e6aae, store=[table=TestAcidGuarantees family=A region=63842f73863e9e7b9587486b122e6aae] 2024-11-24T22:14:37,188 DEBUG [RS:0;bd6592c37bb2:40087-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411246cb35d08b94b451ca806407561820980_63842f73863e9e7b9587486b122e6aae because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=63842f73863e9e7b9587486b122e6aae] 2024-11-24T22:14:37,188 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40087 {}] regionserver.HRegion(8855): Flush requested on 63842f73863e9e7b9587486b122e6aae 2024-11-24T22:14:37,188 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 63842f73863e9e7b9587486b122e6aae 3/3 column families, dataSize=154.31 KB heapSize=430.69 KB 2024-11-24T22:14:37,188 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 63842f73863e9e7b9587486b122e6aae, store=A 2024-11-24T22:14:37,188 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-24T22:14:37,188 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 63842f73863e9e7b9587486b122e6aae, store=B 2024-11-24T22:14:37,189 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-24T22:14:37,189 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 63842f73863e9e7b9587486b122e6aae, store=C 2024-11-24T22:14:37,189 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-24T22:14:37,208 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45929 is added to blk_1073742003_1179 (size=12697) 2024-11-24T22:14:37,215 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411243b844e2044e54833a729059664d61a8b_63842f73863e9e7b9587486b122e6aae is 50, key is test_row_0/A:col10/1732486477186/Put/seqid=0 2024-11-24T22:14:37,217 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45929 is added to blk_1073742004_1180 (size=4469) 2024-11-24T22:14:37,222 INFO [RS:0;bd6592c37bb2:40087-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 63842f73863e9e7b9587486b122e6aae#A#compaction#147 average throughput is 0.48 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-24T22:14:37,222 DEBUG [RS:0;bd6592c37bb2:40087-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/63842f73863e9e7b9587486b122e6aae/.tmp/A/cf64aec7f35742de8b3fb00ac564ba08 is 175, key is test_row_0/A:col10/1732486473479/Put/seqid=0 2024-11-24T22:14:37,222 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40087 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=63842f73863e9e7b9587486b122e6aae, server=bd6592c37bb2,40087,1732486423713 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-24T22:14:37,223 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40087 {}] ipc.CallRunner(138): callId: 105 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40244 deadline: 1732486537210, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=63842f73863e9e7b9587486b122e6aae, server=bd6592c37bb2,40087,1732486423713 2024-11-24T22:14:37,223 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40087 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=63842f73863e9e7b9587486b122e6aae, server=bd6592c37bb2,40087,1732486423713 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-24T22:14:37,223 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732486460633.63842f73863e9e7b9587486b122e6aae., hostname=bd6592c37bb2,40087,1732486423713, seqNum=5 , the old value is region=TestAcidGuarantees,,1732486460633.63842f73863e9e7b9587486b122e6aae., hostname=bd6592c37bb2,40087,1732486423713, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=63842f73863e9e7b9587486b122e6aae, server=bd6592c37bb2,40087,1732486423713 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T22:14:37,223 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732486460633.63842f73863e9e7b9587486b122e6aae., hostname=bd6592c37bb2,40087,1732486423713, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=63842f73863e9e7b9587486b122e6aae, server=bd6592c37bb2,40087,1732486423713 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T22:14:37,223 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40087 {}] ipc.CallRunner(138): callId: 114 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40206 deadline: 1732486537218, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=63842f73863e9e7b9587486b122e6aae, server=bd6592c37bb2,40087,1732486423713 2024-11-24T22:14:37,223 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732486460633.63842f73863e9e7b9587486b122e6aae., hostname=bd6592c37bb2,40087,1732486423713, seqNum=5 because the exception is null or not the one we care about 2024-11-24T22:14:37,224 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732486460633.63842f73863e9e7b9587486b122e6aae., hostname=bd6592c37bb2,40087,1732486423713, seqNum=5 , the old value is region=TestAcidGuarantees,,1732486460633.63842f73863e9e7b9587486b122e6aae., hostname=bd6592c37bb2,40087,1732486423713, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=63842f73863e9e7b9587486b122e6aae, server=bd6592c37bb2,40087,1732486423713 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T22:14:37,224 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732486460633.63842f73863e9e7b9587486b122e6aae., hostname=bd6592c37bb2,40087,1732486423713, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=63842f73863e9e7b9587486b122e6aae, server=bd6592c37bb2,40087,1732486423713 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T22:14:37,224 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732486460633.63842f73863e9e7b9587486b122e6aae., hostname=bd6592c37bb2,40087,1732486423713, seqNum=5 because the exception is null or not the one we care about 2024-11-24T22:14:37,232 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=40087 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=47 2024-11-24T22:14:37,233 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bd6592c37bb2:0-1 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732486460633.63842f73863e9e7b9587486b122e6aae. 2024-11-24T22:14:37,233 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bd6592c37bb2:0-1 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732486460633.63842f73863e9e7b9587486b122e6aae. as already flushing 2024-11-24T22:14:37,233 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bd6592c37bb2:0-1 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732486460633.63842f73863e9e7b9587486b122e6aae. 2024-11-24T22:14:37,233 ERROR [RS_FLUSH_OPERATIONS-regionserver/bd6592c37bb2:0-1 {event_type=RS_FLUSH_REGIONS, pid=47}] handler.RSProcedureHandler(58): pid=47 java.io.IOException: Unable to complete flush {ENCODED => 63842f73863e9e7b9587486b122e6aae, NAME => 'TestAcidGuarantees,,1732486460633.63842f73863e9e7b9587486b122e6aae.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T22:14:37,233 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bd6592c37bb2:0-1 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=47 java.io.IOException: Unable to complete flush {ENCODED => 63842f73863e9e7b9587486b122e6aae, NAME => 'TestAcidGuarantees,,1732486460633.63842f73863e9e7b9587486b122e6aae.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T22:14:37,234 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37701 {}] master.HMaster(4177): Remote procedure failed, pid=47 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 63842f73863e9e7b9587486b122e6aae, NAME => 'TestAcidGuarantees,,1732486460633.63842f73863e9e7b9587486b122e6aae.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 63842f73863e9e7b9587486b122e6aae, NAME => 'TestAcidGuarantees,,1732486460633.63842f73863e9e7b9587486b122e6aae.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T22:14:37,252 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45929 is added to blk_1073742005_1181 (size=17384) 2024-11-24T22:14:37,255 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T22:14:37,260 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411243b844e2044e54833a729059664d61a8b_63842f73863e9e7b9587486b122e6aae to hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411243b844e2044e54833a729059664d61a8b_63842f73863e9e7b9587486b122e6aae 2024-11-24T22:14:37,261 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/63842f73863e9e7b9587486b122e6aae/.tmp/A/ecdf5fa4631f4cc4bfccf8ac391dfc88, store: [table=TestAcidGuarantees family=A region=63842f73863e9e7b9587486b122e6aae] 2024-11-24T22:14:37,262 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/63842f73863e9e7b9587486b122e6aae/.tmp/A/ecdf5fa4631f4cc4bfccf8ac391dfc88 is 175, key is test_row_0/A:col10/1732486477186/Put/seqid=0 2024-11-24T22:14:37,273 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45929 is added to blk_1073742006_1182 (size=31758) 2024-11-24T22:14:37,281 DEBUG [RS:0;bd6592c37bb2:40087-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/63842f73863e9e7b9587486b122e6aae/.tmp/A/cf64aec7f35742de8b3fb00ac564ba08 as hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/63842f73863e9e7b9587486b122e6aae/A/cf64aec7f35742de8b3fb00ac564ba08 2024-11-24T22:14:37,285 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45929 is added to blk_1073742007_1183 (size=48489) 2024-11-24T22:14:37,286 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=259, memsize=51.4 K, hasBloomFilter=true, into tmp file hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/63842f73863e9e7b9587486b122e6aae/.tmp/A/ecdf5fa4631f4cc4bfccf8ac391dfc88 2024-11-24T22:14:37,290 INFO [RS:0;bd6592c37bb2:40087-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 63842f73863e9e7b9587486b122e6aae/A of 63842f73863e9e7b9587486b122e6aae into cf64aec7f35742de8b3fb00ac564ba08(size=31.0 K), total size for store is 31.0 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-24T22:14:37,290 DEBUG [RS:0;bd6592c37bb2:40087-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 63842f73863e9e7b9587486b122e6aae: 2024-11-24T22:14:37,290 INFO [RS:0;bd6592c37bb2:40087-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732486460633.63842f73863e9e7b9587486b122e6aae., storeName=63842f73863e9e7b9587486b122e6aae/A, priority=13, startTime=1732486477150; duration=0sec 2024-11-24T22:14:37,290 DEBUG [RS:0;bd6592c37bb2:40087-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-24T22:14:37,290 DEBUG [RS:0;bd6592c37bb2:40087-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 63842f73863e9e7b9587486b122e6aae:A 2024-11-24T22:14:37,293 DEBUG [RS:0;bd6592c37bb2:40087-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-24T22:14:37,295 DEBUG [RS:0;bd6592c37bb2:40087-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 34503 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-24T22:14:37,295 DEBUG [RS:0;bd6592c37bb2:40087-shortCompactions-0 {}] regionserver.HStore(1541): 63842f73863e9e7b9587486b122e6aae/C is initiating minor compaction (all files) 2024-11-24T22:14:37,295 INFO [RS:0;bd6592c37bb2:40087-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 63842f73863e9e7b9587486b122e6aae/C in TestAcidGuarantees,,1732486460633.63842f73863e9e7b9587486b122e6aae. 2024-11-24T22:14:37,296 INFO [RS:0;bd6592c37bb2:40087-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/63842f73863e9e7b9587486b122e6aae/C/4923864c6fda45d6b701d9e800bd4c46, hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/63842f73863e9e7b9587486b122e6aae/C/58561c0c991f467ca37137ea8d0cf3df, hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/63842f73863e9e7b9587486b122e6aae/C/f9001c2421d641e1a3a5fa501a7bf407] into tmpdir=hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/63842f73863e9e7b9587486b122e6aae/.tmp, totalSize=33.7 K 2024-11-24T22:14:37,299 DEBUG [RS:0;bd6592c37bb2:40087-shortCompactions-0 {}] compactions.Compactor(225): Compacting 4923864c6fda45d6b701d9e800bd4c46, keycount=150, bloomtype=ROW, size=12.3 K, encoding=NONE, compression=NONE, seqNum=194, earliestPutTs=1732486471251 2024-11-24T22:14:37,299 DEBUG [RS:0;bd6592c37bb2:40087-shortCompactions-0 {}] compactions.Compactor(225): Compacting 58561c0c991f467ca37137ea8d0cf3df, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=209, earliestPutTs=1732486473447 2024-11-24T22:14:37,300 DEBUG [RS:0;bd6592c37bb2:40087-shortCompactions-0 {}] compactions.Compactor(225): Compacting f9001c2421d641e1a3a5fa501a7bf407, keycount=100, bloomtype=ROW, size=9.5 K, encoding=NONE, compression=NONE, seqNum=220, earliestPutTs=1732486475776 2024-11-24T22:14:37,307 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/63842f73863e9e7b9587486b122e6aae/.tmp/B/81e2cda377e2474c933a28ab8f351261 is 50, key is test_row_0/B:col10/1732486477186/Put/seqid=0 2024-11-24T22:14:37,318 INFO [RS:0;bd6592c37bb2:40087-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 63842f73863e9e7b9587486b122e6aae#C#compaction#151 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-24T22:14:37,319 DEBUG [RS:0;bd6592c37bb2:40087-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/63842f73863e9e7b9587486b122e6aae/.tmp/C/17307ba8f7ec483e8fc9c7d11b47d475 is 50, key is test_row_0/C:col10/1732486473479/Put/seqid=0 2024-11-24T22:14:37,323 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45929 is added to blk_1073742008_1184 (size=12201) 2024-11-24T22:14:37,324 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=51.44 KB at sequenceid=259 (bloomFilter=true), to=hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/63842f73863e9e7b9587486b122e6aae/.tmp/B/81e2cda377e2474c933a28ab8f351261 2024-11-24T22:14:37,335 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40087 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=63842f73863e9e7b9587486b122e6aae, server=bd6592c37bb2,40087,1732486423713 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-24T22:14:37,335 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40087 {}] ipc.CallRunner(138): callId: 106 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40244 deadline: 1732486537330, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=63842f73863e9e7b9587486b122e6aae, server=bd6592c37bb2,40087,1732486423713 2024-11-24T22:14:37,335 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40087 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=63842f73863e9e7b9587486b122e6aae, server=bd6592c37bb2,40087,1732486423713 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-24T22:14:37,335 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732486460633.63842f73863e9e7b9587486b122e6aae., hostname=bd6592c37bb2,40087,1732486423713, seqNum=5 , the old value is region=TestAcidGuarantees,,1732486460633.63842f73863e9e7b9587486b122e6aae., hostname=bd6592c37bb2,40087,1732486423713, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=63842f73863e9e7b9587486b122e6aae, server=bd6592c37bb2,40087,1732486423713 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T22:14:37,335 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732486460633.63842f73863e9e7b9587486b122e6aae., hostname=bd6592c37bb2,40087,1732486423713, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=63842f73863e9e7b9587486b122e6aae, server=bd6592c37bb2,40087,1732486423713 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T22:14:37,335 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40087 {}] ipc.CallRunner(138): callId: 115 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40206 deadline: 1732486537331, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=63842f73863e9e7b9587486b122e6aae, server=bd6592c37bb2,40087,1732486423713 2024-11-24T22:14:37,335 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732486460633.63842f73863e9e7b9587486b122e6aae., hostname=bd6592c37bb2,40087,1732486423713, seqNum=5 because the exception is null or not the one we care about 2024-11-24T22:14:37,336 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732486460633.63842f73863e9e7b9587486b122e6aae., hostname=bd6592c37bb2,40087,1732486423713, seqNum=5 , the old value is region=TestAcidGuarantees,,1732486460633.63842f73863e9e7b9587486b122e6aae., hostname=bd6592c37bb2,40087,1732486423713, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=63842f73863e9e7b9587486b122e6aae, server=bd6592c37bb2,40087,1732486423713 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T22:14:37,336 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732486460633.63842f73863e9e7b9587486b122e6aae., hostname=bd6592c37bb2,40087,1732486423713, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=63842f73863e9e7b9587486b122e6aae, server=bd6592c37bb2,40087,1732486423713 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T22:14:37,336 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732486460633.63842f73863e9e7b9587486b122e6aae., hostname=bd6592c37bb2,40087,1732486423713, seqNum=5 because the exception is null or not the one we care about 2024-11-24T22:14:37,347 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45929 is added to blk_1073742009_1185 (size=12697) 2024-11-24T22:14:37,355 DEBUG [RS:0;bd6592c37bb2:40087-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/63842f73863e9e7b9587486b122e6aae/.tmp/C/17307ba8f7ec483e8fc9c7d11b47d475 as hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/63842f73863e9e7b9587486b122e6aae/C/17307ba8f7ec483e8fc9c7d11b47d475 2024-11-24T22:14:37,364 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/63842f73863e9e7b9587486b122e6aae/.tmp/C/c0250f2ef6c147c1ac6f1adc8c0836d6 is 50, key is test_row_0/C:col10/1732486477186/Put/seqid=0 2024-11-24T22:14:37,367 INFO [RS:0;bd6592c37bb2:40087-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 63842f73863e9e7b9587486b122e6aae/C of 63842f73863e9e7b9587486b122e6aae into 17307ba8f7ec483e8fc9c7d11b47d475(size=12.4 K), total size for store is 12.4 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-24T22:14:37,367 DEBUG [RS:0;bd6592c37bb2:40087-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 63842f73863e9e7b9587486b122e6aae: 2024-11-24T22:14:37,367 INFO [RS:0;bd6592c37bb2:40087-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732486460633.63842f73863e9e7b9587486b122e6aae., storeName=63842f73863e9e7b9587486b122e6aae/C, priority=13, startTime=1732486477150; duration=0sec 2024-11-24T22:14:37,367 DEBUG [RS:0;bd6592c37bb2:40087-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-24T22:14:37,367 DEBUG [RS:0;bd6592c37bb2:40087-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 63842f73863e9e7b9587486b122e6aae:C 2024-11-24T22:14:37,391 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=40087 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=47 2024-11-24T22:14:37,392 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bd6592c37bb2:0-2 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732486460633.63842f73863e9e7b9587486b122e6aae. 2024-11-24T22:14:37,392 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bd6592c37bb2:0-2 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732486460633.63842f73863e9e7b9587486b122e6aae. as already flushing 2024-11-24T22:14:37,392 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bd6592c37bb2:0-2 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732486460633.63842f73863e9e7b9587486b122e6aae. 2024-11-24T22:14:37,392 ERROR [RS_FLUSH_OPERATIONS-regionserver/bd6592c37bb2:0-2 {event_type=RS_FLUSH_REGIONS, pid=47}] handler.RSProcedureHandler(58): pid=47 java.io.IOException: Unable to complete flush {ENCODED => 63842f73863e9e7b9587486b122e6aae, NAME => 'TestAcidGuarantees,,1732486460633.63842f73863e9e7b9587486b122e6aae.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T22:14:37,392 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bd6592c37bb2:0-2 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=47 java.io.IOException: Unable to complete flush {ENCODED => 63842f73863e9e7b9587486b122e6aae, NAME => 'TestAcidGuarantees,,1732486460633.63842f73863e9e7b9587486b122e6aae.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T22:14:37,393 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37701 {}] master.HMaster(4177): Remote procedure failed, pid=47 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 63842f73863e9e7b9587486b122e6aae, NAME => 'TestAcidGuarantees,,1732486460633.63842f73863e9e7b9587486b122e6aae.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 63842f73863e9e7b9587486b122e6aae, NAME => 'TestAcidGuarantees,,1732486460633.63842f73863e9e7b9587486b122e6aae.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T22:14:37,426 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45929 is added to blk_1073742010_1186 (size=12201) 2024-11-24T22:14:37,427 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=51.44 KB at sequenceid=259 (bloomFilter=true), to=hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/63842f73863e9e7b9587486b122e6aae/.tmp/C/c0250f2ef6c147c1ac6f1adc8c0836d6 2024-11-24T22:14:37,433 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/63842f73863e9e7b9587486b122e6aae/.tmp/A/ecdf5fa4631f4cc4bfccf8ac391dfc88 as hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/63842f73863e9e7b9587486b122e6aae/A/ecdf5fa4631f4cc4bfccf8ac391dfc88 2024-11-24T22:14:37,441 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/63842f73863e9e7b9587486b122e6aae/A/ecdf5fa4631f4cc4bfccf8ac391dfc88, entries=250, sequenceid=259, filesize=47.4 K 2024-11-24T22:14:37,442 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/63842f73863e9e7b9587486b122e6aae/.tmp/B/81e2cda377e2474c933a28ab8f351261 as hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/63842f73863e9e7b9587486b122e6aae/B/81e2cda377e2474c933a28ab8f351261 2024-11-24T22:14:37,449 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/63842f73863e9e7b9587486b122e6aae/B/81e2cda377e2474c933a28ab8f351261, entries=150, sequenceid=259, filesize=11.9 K 2024-11-24T22:14:37,452 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/63842f73863e9e7b9587486b122e6aae/.tmp/C/c0250f2ef6c147c1ac6f1adc8c0836d6 as hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/63842f73863e9e7b9587486b122e6aae/C/c0250f2ef6c147c1ac6f1adc8c0836d6 2024-11-24T22:14:37,457 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/63842f73863e9e7b9587486b122e6aae/C/c0250f2ef6c147c1ac6f1adc8c0836d6, entries=150, sequenceid=259, filesize=11.9 K 2024-11-24T22:14:37,459 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~154.31 KB/158010, heapSize ~430.64 KB/440976, currentSize=33.54 KB/34350 for 63842f73863e9e7b9587486b122e6aae in 271ms, sequenceid=259, compaction requested=false 2024-11-24T22:14:37,459 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 63842f73863e9e7b9587486b122e6aae: 2024-11-24T22:14:37,546 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=40087 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=47 2024-11-24T22:14:37,546 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bd6592c37bb2:0-0 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732486460633.63842f73863e9e7b9587486b122e6aae. 2024-11-24T22:14:37,547 INFO [RS_FLUSH_OPERATIONS-regionserver/bd6592c37bb2:0-0 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.HRegion(2902): Flushing 63842f73863e9e7b9587486b122e6aae 3/3 column families, dataSize=40.25 KB heapSize=120.28 KB 2024-11-24T22:14:37,551 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40087 {}] regionserver.HRegion(8855): Flush requested on 63842f73863e9e7b9587486b122e6aae 2024-11-24T22:14:37,551 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bd6592c37bb2:0-0 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 63842f73863e9e7b9587486b122e6aae, store=A 2024-11-24T22:14:37,551 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bd6592c37bb2:0-0 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-24T22:14:37,551 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bd6592c37bb2:0-0 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 63842f73863e9e7b9587486b122e6aae, store=B 2024-11-24T22:14:37,551 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bd6592c37bb2:0-0 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-24T22:14:37,551 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bd6592c37bb2:0-0 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 63842f73863e9e7b9587486b122e6aae, store=C 2024-11-24T22:14:37,551 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bd6592c37bb2:0-0 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-24T22:14:37,551 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732486460633.63842f73863e9e7b9587486b122e6aae. as already flushing 2024-11-24T22:14:37,563 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bd6592c37bb2:0-0 {event_type=RS_FLUSH_REGIONS, pid=47}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241124c53264bab37e4ca68e6dc4d5f7177c66_63842f73863e9e7b9587486b122e6aae is 50, key is test_row_0/A:col10/1732486477216/Put/seqid=0 2024-11-24T22:14:37,591 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37701 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=46 2024-11-24T22:14:37,606 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45929 is added to blk_1073742011_1187 (size=12454) 2024-11-24T22:14:37,608 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bd6592c37bb2:0-0 {event_type=RS_FLUSH_REGIONS, pid=47}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T22:14:37,612 DEBUG [RS:0;bd6592c37bb2:40087-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/63842f73863e9e7b9587486b122e6aae/.tmp/B/455dcb4f2aca48b98940fe2ebddff3fb as hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/63842f73863e9e7b9587486b122e6aae/B/455dcb4f2aca48b98940fe2ebddff3fb 2024-11-24T22:14:37,616 INFO [RS_FLUSH_OPERATIONS-regionserver/bd6592c37bb2:0-0 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241124c53264bab37e4ca68e6dc4d5f7177c66_63842f73863e9e7b9587486b122e6aae to hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241124c53264bab37e4ca68e6dc4d5f7177c66_63842f73863e9e7b9587486b122e6aae 2024-11-24T22:14:37,618 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bd6592c37bb2:0-0 {event_type=RS_FLUSH_REGIONS, pid=47}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/63842f73863e9e7b9587486b122e6aae/.tmp/A/a5c5777a76c04dc7a2b81e81d7e613eb, store: [table=TestAcidGuarantees family=A region=63842f73863e9e7b9587486b122e6aae] 2024-11-24T22:14:37,619 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bd6592c37bb2:0-0 {event_type=RS_FLUSH_REGIONS, pid=47}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/63842f73863e9e7b9587486b122e6aae/.tmp/A/a5c5777a76c04dc7a2b81e81d7e613eb is 175, key is test_row_0/A:col10/1732486477216/Put/seqid=0 2024-11-24T22:14:37,620 INFO [RS:0;bd6592c37bb2:40087-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 63842f73863e9e7b9587486b122e6aae/B of 63842f73863e9e7b9587486b122e6aae into 455dcb4f2aca48b98940fe2ebddff3fb(size=12.4 K), total size for store is 24.3 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-24T22:14:37,620 DEBUG [RS:0;bd6592c37bb2:40087-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 63842f73863e9e7b9587486b122e6aae: 2024-11-24T22:14:37,621 INFO [RS:0;bd6592c37bb2:40087-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732486460633.63842f73863e9e7b9587486b122e6aae., storeName=63842f73863e9e7b9587486b122e6aae/B, priority=13, startTime=1732486477150; duration=0sec 2024-11-24T22:14:37,621 DEBUG [RS:0;bd6592c37bb2:40087-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-24T22:14:37,621 DEBUG [RS:0;bd6592c37bb2:40087-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 63842f73863e9e7b9587486b122e6aae:B 2024-11-24T22:14:37,629 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45929 is added to blk_1073742012_1188 (size=31255) 2024-11-24T22:14:37,631 INFO [RS_FLUSH_OPERATIONS-regionserver/bd6592c37bb2:0-0 {event_type=RS_FLUSH_REGIONS, pid=47}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=271, memsize=15.7 K, hasBloomFilter=true, into tmp file hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/63842f73863e9e7b9587486b122e6aae/.tmp/A/a5c5777a76c04dc7a2b81e81d7e613eb 2024-11-24T22:14:37,645 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/bd6592c37bb2:0-9 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=259, cellSize=11855, totalHeapSize=27176, min timestamp=1732486477614, max timestamp=1732486477641Num uniques 859; 2024-11-24T22:14:37,645 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/bd6592c37bb2:0-9 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=11855, suffix heap size=117064,new segment heap size=27176 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=259 2024-11-24T22:14:37,645 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/bd6592c37bb2:0-4 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=259, cellSize=11855, totalHeapSize=27176, min timestamp=1732486477614, max timestamp=1732486477641Num uniques 859; 2024-11-24T22:14:37,645 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/bd6592c37bb2:0-4 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=11855, suffix heap size=117064,new segment heap size=27176 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=259 2024-11-24T22:14:37,646 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/bd6592c37bb2:0-6 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=259, cellSize=11855, totalHeapSize=27176, min timestamp=1732486477614, max timestamp=1732486477641Num uniques 859; 2024-11-24T22:14:37,646 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/bd6592c37bb2:0-6 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=11855, suffix heap size=117064,new segment heap size=27176 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=259 2024-11-24T22:14:37,647 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bd6592c37bb2:0-0 {event_type=RS_FLUSH_REGIONS, pid=47}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/63842f73863e9e7b9587486b122e6aae/.tmp/B/09eff7dc8cda475b93df975f70ffcc4a is 50, key is test_row_0/B:col10/1732486477216/Put/seqid=0 2024-11-24T22:14:37,655 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45929 is added to blk_1073742013_1189 (size=12301) 2024-11-24T22:14:37,658 INFO [RS_FLUSH_OPERATIONS-regionserver/bd6592c37bb2:0-0 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=271 (bloomFilter=true), to=hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/63842f73863e9e7b9587486b122e6aae/.tmp/B/09eff7dc8cda475b93df975f70ffcc4a 2024-11-24T22:14:37,702 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bd6592c37bb2:0-0 {event_type=RS_FLUSH_REGIONS, pid=47}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/63842f73863e9e7b9587486b122e6aae/.tmp/C/253f3fb757454dfcbb4d4a532093bc69 is 50, key is test_row_0/C:col10/1732486477216/Put/seqid=0 2024-11-24T22:14:37,710 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40087 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=63842f73863e9e7b9587486b122e6aae, server=bd6592c37bb2,40087,1732486423713 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-24T22:14:37,711 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40087 {}] ipc.CallRunner(138): callId: 124 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40244 deadline: 1732486537705, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=63842f73863e9e7b9587486b122e6aae, server=bd6592c37bb2,40087,1732486423713 2024-11-24T22:14:37,711 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732486460633.63842f73863e9e7b9587486b122e6aae., hostname=bd6592c37bb2,40087,1732486423713, seqNum=5 , the old value is region=TestAcidGuarantees,,1732486460633.63842f73863e9e7b9587486b122e6aae., hostname=bd6592c37bb2,40087,1732486423713, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=63842f73863e9e7b9587486b122e6aae, server=bd6592c37bb2,40087,1732486423713 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T22:14:37,711 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732486460633.63842f73863e9e7b9587486b122e6aae., hostname=bd6592c37bb2,40087,1732486423713, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=63842f73863e9e7b9587486b122e6aae, server=bd6592c37bb2,40087,1732486423713 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T22:14:37,711 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732486460633.63842f73863e9e7b9587486b122e6aae., hostname=bd6592c37bb2,40087,1732486423713, seqNum=5 because the exception is null or not the one we care about 2024-11-24T22:14:37,715 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40087 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=63842f73863e9e7b9587486b122e6aae, server=bd6592c37bb2,40087,1732486423713 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-24T22:14:37,715 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40087 {}] ipc.CallRunner(138): callId: 133 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40206 deadline: 1732486537708, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=63842f73863e9e7b9587486b122e6aae, server=bd6592c37bb2,40087,1732486423713 2024-11-24T22:14:37,716 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732486460633.63842f73863e9e7b9587486b122e6aae., hostname=bd6592c37bb2,40087,1732486423713, seqNum=5 , the old value is region=TestAcidGuarantees,,1732486460633.63842f73863e9e7b9587486b122e6aae., hostname=bd6592c37bb2,40087,1732486423713, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=63842f73863e9e7b9587486b122e6aae, server=bd6592c37bb2,40087,1732486423713 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T22:14:37,716 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732486460633.63842f73863e9e7b9587486b122e6aae., hostname=bd6592c37bb2,40087,1732486423713, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=63842f73863e9e7b9587486b122e6aae, server=bd6592c37bb2,40087,1732486423713 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T22:14:37,716 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732486460633.63842f73863e9e7b9587486b122e6aae., hostname=bd6592c37bb2,40087,1732486423713, seqNum=5 because the exception is null or not the one we care about 2024-11-24T22:14:37,736 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45929 is added to blk_1073742014_1190 (size=12301) 2024-11-24T22:14:37,826 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40087 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=63842f73863e9e7b9587486b122e6aae, server=bd6592c37bb2,40087,1732486423713 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-24T22:14:37,826 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40087 {}] ipc.CallRunner(138): callId: 125 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40244 deadline: 1732486537821, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=63842f73863e9e7b9587486b122e6aae, server=bd6592c37bb2,40087,1732486423713 2024-11-24T22:14:37,827 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732486460633.63842f73863e9e7b9587486b122e6aae., hostname=bd6592c37bb2,40087,1732486423713, seqNum=5 , the old value is region=TestAcidGuarantees,,1732486460633.63842f73863e9e7b9587486b122e6aae., hostname=bd6592c37bb2,40087,1732486423713, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=63842f73863e9e7b9587486b122e6aae, server=bd6592c37bb2,40087,1732486423713 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T22:14:37,827 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40087 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=63842f73863e9e7b9587486b122e6aae, server=bd6592c37bb2,40087,1732486423713 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-24T22:14:37,827 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732486460633.63842f73863e9e7b9587486b122e6aae., hostname=bd6592c37bb2,40087,1732486423713, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=63842f73863e9e7b9587486b122e6aae, server=bd6592c37bb2,40087,1732486423713 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T22:14:37,827 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732486460633.63842f73863e9e7b9587486b122e6aae., hostname=bd6592c37bb2,40087,1732486423713, seqNum=5 because the exception is null or not the one we care about 2024-11-24T22:14:37,827 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40087 {}] ipc.CallRunner(138): callId: 134 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40206 deadline: 1732486537821, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=63842f73863e9e7b9587486b122e6aae, server=bd6592c37bb2,40087,1732486423713 2024-11-24T22:14:37,827 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732486460633.63842f73863e9e7b9587486b122e6aae., hostname=bd6592c37bb2,40087,1732486423713, seqNum=5 , the old value is region=TestAcidGuarantees,,1732486460633.63842f73863e9e7b9587486b122e6aae., hostname=bd6592c37bb2,40087,1732486423713, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=63842f73863e9e7b9587486b122e6aae, server=bd6592c37bb2,40087,1732486423713 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T22:14:37,828 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732486460633.63842f73863e9e7b9587486b122e6aae., hostname=bd6592c37bb2,40087,1732486423713, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=63842f73863e9e7b9587486b122e6aae, server=bd6592c37bb2,40087,1732486423713 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T22:14:37,828 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732486460633.63842f73863e9e7b9587486b122e6aae., hostname=bd6592c37bb2,40087,1732486423713, seqNum=5 because the exception is null or not the one we care about 2024-11-24T22:14:38,032 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40087 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=63842f73863e9e7b9587486b122e6aae, server=bd6592c37bb2,40087,1732486423713 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-24T22:14:38,032 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40087 {}] ipc.CallRunner(138): callId: 126 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40244 deadline: 1732486538031, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=63842f73863e9e7b9587486b122e6aae, server=bd6592c37bb2,40087,1732486423713 2024-11-24T22:14:38,032 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40087 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=63842f73863e9e7b9587486b122e6aae, server=bd6592c37bb2,40087,1732486423713 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-24T22:14:38,033 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40087 {}] ipc.CallRunner(138): callId: 135 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40206 deadline: 1732486538031, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=63842f73863e9e7b9587486b122e6aae, server=bd6592c37bb2,40087,1732486423713 2024-11-24T22:14:38,033 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732486460633.63842f73863e9e7b9587486b122e6aae., hostname=bd6592c37bb2,40087,1732486423713, seqNum=5 , the old value is region=TestAcidGuarantees,,1732486460633.63842f73863e9e7b9587486b122e6aae., hostname=bd6592c37bb2,40087,1732486423713, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=63842f73863e9e7b9587486b122e6aae, server=bd6592c37bb2,40087,1732486423713 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T22:14:38,033 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732486460633.63842f73863e9e7b9587486b122e6aae., hostname=bd6592c37bb2,40087,1732486423713, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=63842f73863e9e7b9587486b122e6aae, server=bd6592c37bb2,40087,1732486423713 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T22:14:38,033 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732486460633.63842f73863e9e7b9587486b122e6aae., hostname=bd6592c37bb2,40087,1732486423713, seqNum=5 , the old value is region=TestAcidGuarantees,,1732486460633.63842f73863e9e7b9587486b122e6aae., hostname=bd6592c37bb2,40087,1732486423713, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=63842f73863e9e7b9587486b122e6aae, server=bd6592c37bb2,40087,1732486423713 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T22:14:38,033 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732486460633.63842f73863e9e7b9587486b122e6aae., hostname=bd6592c37bb2,40087,1732486423713, seqNum=5 because the exception is null or not the one we care about 2024-11-24T22:14:38,033 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732486460633.63842f73863e9e7b9587486b122e6aae., hostname=bd6592c37bb2,40087,1732486423713, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=63842f73863e9e7b9587486b122e6aae, server=bd6592c37bb2,40087,1732486423713 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T22:14:38,033 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732486460633.63842f73863e9e7b9587486b122e6aae., hostname=bd6592c37bb2,40087,1732486423713, seqNum=5 because the exception is null or not the one we care about 2024-11-24T22:14:38,137 INFO [RS_FLUSH_OPERATIONS-regionserver/bd6592c37bb2:0-0 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=271 (bloomFilter=true), to=hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/63842f73863e9e7b9587486b122e6aae/.tmp/C/253f3fb757454dfcbb4d4a532093bc69 2024-11-24T22:14:38,142 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bd6592c37bb2:0-0 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/63842f73863e9e7b9587486b122e6aae/.tmp/A/a5c5777a76c04dc7a2b81e81d7e613eb as hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/63842f73863e9e7b9587486b122e6aae/A/a5c5777a76c04dc7a2b81e81d7e613eb 2024-11-24T22:14:38,146 INFO [RS_FLUSH_OPERATIONS-regionserver/bd6592c37bb2:0-0 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/63842f73863e9e7b9587486b122e6aae/A/a5c5777a76c04dc7a2b81e81d7e613eb, entries=150, sequenceid=271, filesize=30.5 K 2024-11-24T22:14:38,147 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bd6592c37bb2:0-0 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/63842f73863e9e7b9587486b122e6aae/.tmp/B/09eff7dc8cda475b93df975f70ffcc4a as hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/63842f73863e9e7b9587486b122e6aae/B/09eff7dc8cda475b93df975f70ffcc4a 2024-11-24T22:14:38,151 INFO [RS_FLUSH_OPERATIONS-regionserver/bd6592c37bb2:0-0 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/63842f73863e9e7b9587486b122e6aae/B/09eff7dc8cda475b93df975f70ffcc4a, entries=150, sequenceid=271, filesize=12.0 K 2024-11-24T22:14:38,152 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bd6592c37bb2:0-0 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/63842f73863e9e7b9587486b122e6aae/.tmp/C/253f3fb757454dfcbb4d4a532093bc69 as hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/63842f73863e9e7b9587486b122e6aae/C/253f3fb757454dfcbb4d4a532093bc69 2024-11-24T22:14:38,159 INFO [RS_FLUSH_OPERATIONS-regionserver/bd6592c37bb2:0-0 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/63842f73863e9e7b9587486b122e6aae/C/253f3fb757454dfcbb4d4a532093bc69, entries=150, sequenceid=271, filesize=12.0 K 2024-11-24T22:14:38,160 INFO [RS_FLUSH_OPERATIONS-regionserver/bd6592c37bb2:0-0 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.HRegion(3140): Finished flush of dataSize ~46.96 KB/48090, heapSize ~140.16 KB/143520, currentSize=134.18 KB/137400 for 63842f73863e9e7b9587486b122e6aae in 614ms, sequenceid=271, compaction requested=true 2024-11-24T22:14:38,160 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bd6592c37bb2:0-0 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.HRegion(2603): Flush status journal for 63842f73863e9e7b9587486b122e6aae: 2024-11-24T22:14:38,160 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bd6592c37bb2:0-0 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732486460633.63842f73863e9e7b9587486b122e6aae. 2024-11-24T22:14:38,160 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bd6592c37bb2:0-0 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=47 2024-11-24T22:14:38,161 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37701 {}] master.HMaster(4169): Remote procedure done, pid=47 2024-11-24T22:14:38,165 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=47, resume processing ppid=46 2024-11-24T22:14:38,165 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=47, ppid=46, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.7050 sec 2024-11-24T22:14:38,167 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=46, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=46, table=TestAcidGuarantees in 1.7130 sec 2024-11-24T22:14:38,333 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40087 {}] regionserver.HRegion(8855): Flush requested on 63842f73863e9e7b9587486b122e6aae 2024-11-24T22:14:38,333 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 63842f73863e9e7b9587486b122e6aae 3/3 column families, dataSize=140.89 KB heapSize=395.53 KB 2024-11-24T22:14:38,333 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 63842f73863e9e7b9587486b122e6aae, store=A 2024-11-24T22:14:38,334 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-24T22:14:38,334 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 63842f73863e9e7b9587486b122e6aae, store=B 2024-11-24T22:14:38,334 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-24T22:14:38,334 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 63842f73863e9e7b9587486b122e6aae, store=C 2024-11-24T22:14:38,334 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-24T22:14:38,342 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241124dff701258fc247db9f301f1ffefc5958_63842f73863e9e7b9587486b122e6aae is 50, key is test_row_0/A:col10/1732486478332/Put/seqid=0 2024-11-24T22:14:38,351 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45929 is added to blk_1073742015_1191 (size=14994) 2024-11-24T22:14:38,357 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40087 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=63842f73863e9e7b9587486b122e6aae, server=bd6592c37bb2,40087,1732486423713 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-24T22:14:38,358 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40087 {}] ipc.CallRunner(138): callId: 54 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40202 deadline: 1732486538349, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=63842f73863e9e7b9587486b122e6aae, server=bd6592c37bb2,40087,1732486423713 2024-11-24T22:14:38,358 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40087 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=63842f73863e9e7b9587486b122e6aae, server=bd6592c37bb2,40087,1732486423713 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-24T22:14:38,358 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40087 {}] ipc.CallRunner(138): callId: 129 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40244 deadline: 1732486538350, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=63842f73863e9e7b9587486b122e6aae, server=bd6592c37bb2,40087,1732486423713 2024-11-24T22:14:38,358 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732486460633.63842f73863e9e7b9587486b122e6aae., hostname=bd6592c37bb2,40087,1732486423713, seqNum=5 , the old value is region=TestAcidGuarantees,,1732486460633.63842f73863e9e7b9587486b122e6aae., hostname=bd6592c37bb2,40087,1732486423713, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=63842f73863e9e7b9587486b122e6aae, server=bd6592c37bb2,40087,1732486423713 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T22:14:38,358 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732486460633.63842f73863e9e7b9587486b122e6aae., hostname=bd6592c37bb2,40087,1732486423713, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=63842f73863e9e7b9587486b122e6aae, server=bd6592c37bb2,40087,1732486423713 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T22:14:38,358 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732486460633.63842f73863e9e7b9587486b122e6aae., hostname=bd6592c37bb2,40087,1732486423713, seqNum=5 because the exception is null or not the one we care about 2024-11-24T22:14:38,358 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732486460633.63842f73863e9e7b9587486b122e6aae., hostname=bd6592c37bb2,40087,1732486423713, seqNum=5 , the old value is region=TestAcidGuarantees,,1732486460633.63842f73863e9e7b9587486b122e6aae., hostname=bd6592c37bb2,40087,1732486423713, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=63842f73863e9e7b9587486b122e6aae, server=bd6592c37bb2,40087,1732486423713 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T22:14:38,358 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40087 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=63842f73863e9e7b9587486b122e6aae, server=bd6592c37bb2,40087,1732486423713 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-24T22:14:38,358 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732486460633.63842f73863e9e7b9587486b122e6aae., hostname=bd6592c37bb2,40087,1732486423713, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=63842f73863e9e7b9587486b122e6aae, server=bd6592c37bb2,40087,1732486423713 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T22:14:38,358 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732486460633.63842f73863e9e7b9587486b122e6aae., hostname=bd6592c37bb2,40087,1732486423713, seqNum=5 because the exception is null or not the one we care about 2024-11-24T22:14:38,358 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40087 {}] ipc.CallRunner(138): callId: 138 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40206 deadline: 1732486538351, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=63842f73863e9e7b9587486b122e6aae, server=bd6592c37bb2,40087,1732486423713 2024-11-24T22:14:38,359 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732486460633.63842f73863e9e7b9587486b122e6aae., hostname=bd6592c37bb2,40087,1732486423713, seqNum=5 , the old value is region=TestAcidGuarantees,,1732486460633.63842f73863e9e7b9587486b122e6aae., hostname=bd6592c37bb2,40087,1732486423713, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=63842f73863e9e7b9587486b122e6aae, server=bd6592c37bb2,40087,1732486423713 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T22:14:38,359 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732486460633.63842f73863e9e7b9587486b122e6aae., hostname=bd6592c37bb2,40087,1732486423713, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=63842f73863e9e7b9587486b122e6aae, server=bd6592c37bb2,40087,1732486423713 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T22:14:38,359 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732486460633.63842f73863e9e7b9587486b122e6aae., hostname=bd6592c37bb2,40087,1732486423713, seqNum=5 because the exception is null or not the one we care about 2024-11-24T22:14:38,466 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40087 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=63842f73863e9e7b9587486b122e6aae, server=bd6592c37bb2,40087,1732486423713 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-24T22:14:38,466 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40087 {}] ipc.CallRunner(138): callId: 55 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40202 deadline: 1732486538460, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=63842f73863e9e7b9587486b122e6aae, server=bd6592c37bb2,40087,1732486423713 2024-11-24T22:14:38,466 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40087 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=63842f73863e9e7b9587486b122e6aae, server=bd6592c37bb2,40087,1732486423713 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-24T22:14:38,467 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40087 {}] ipc.CallRunner(138): callId: 139 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40206 deadline: 1732486538461, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=63842f73863e9e7b9587486b122e6aae, server=bd6592c37bb2,40087,1732486423713 2024-11-24T22:14:38,467 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732486460633.63842f73863e9e7b9587486b122e6aae., hostname=bd6592c37bb2,40087,1732486423713, seqNum=5 , the old value is region=TestAcidGuarantees,,1732486460633.63842f73863e9e7b9587486b122e6aae., hostname=bd6592c37bb2,40087,1732486423713, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=63842f73863e9e7b9587486b122e6aae, server=bd6592c37bb2,40087,1732486423713 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T22:14:38,467 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732486460633.63842f73863e9e7b9587486b122e6aae., hostname=bd6592c37bb2,40087,1732486423713, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=63842f73863e9e7b9587486b122e6aae, server=bd6592c37bb2,40087,1732486423713 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T22:14:38,467 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732486460633.63842f73863e9e7b9587486b122e6aae., hostname=bd6592c37bb2,40087,1732486423713, seqNum=5 because the exception is null or not the one we care about 2024-11-24T22:14:38,467 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40087 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=63842f73863e9e7b9587486b122e6aae, server=bd6592c37bb2,40087,1732486423713 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-24T22:14:38,467 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732486460633.63842f73863e9e7b9587486b122e6aae., hostname=bd6592c37bb2,40087,1732486423713, seqNum=5 , the old value is region=TestAcidGuarantees,,1732486460633.63842f73863e9e7b9587486b122e6aae., hostname=bd6592c37bb2,40087,1732486423713, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=63842f73863e9e7b9587486b122e6aae, server=bd6592c37bb2,40087,1732486423713 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T22:14:38,467 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732486460633.63842f73863e9e7b9587486b122e6aae., hostname=bd6592c37bb2,40087,1732486423713, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=63842f73863e9e7b9587486b122e6aae, server=bd6592c37bb2,40087,1732486423713 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T22:14:38,467 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40087 {}] ipc.CallRunner(138): callId: 130 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40244 deadline: 1732486538461, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=63842f73863e9e7b9587486b122e6aae, server=bd6592c37bb2,40087,1732486423713 2024-11-24T22:14:38,467 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732486460633.63842f73863e9e7b9587486b122e6aae., hostname=bd6592c37bb2,40087,1732486423713, seqNum=5 because the exception is null or not the one we care about 2024-11-24T22:14:38,467 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732486460633.63842f73863e9e7b9587486b122e6aae., hostname=bd6592c37bb2,40087,1732486423713, seqNum=5 , the old value is region=TestAcidGuarantees,,1732486460633.63842f73863e9e7b9587486b122e6aae., hostname=bd6592c37bb2,40087,1732486423713, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=63842f73863e9e7b9587486b122e6aae, server=bd6592c37bb2,40087,1732486423713 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T22:14:38,467 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732486460633.63842f73863e9e7b9587486b122e6aae., hostname=bd6592c37bb2,40087,1732486423713, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=63842f73863e9e7b9587486b122e6aae, server=bd6592c37bb2,40087,1732486423713 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T22:14:38,467 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732486460633.63842f73863e9e7b9587486b122e6aae., hostname=bd6592c37bb2,40087,1732486423713, seqNum=5 because the exception is null or not the one we care about 2024-11-24T22:14:38,601 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37701 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=46 2024-11-24T22:14:38,602 INFO [RPCClient-NioEventLoopGroup-4-7 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-24T22:14:38,604 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37701 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-24T22:14:38,605 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37701 {}] procedure2.ProcedureExecutor(1139): Stored pid=48, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=48, table=TestAcidGuarantees 2024-11-24T22:14:38,606 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=48, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=48, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-24T22:14:38,608 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=48, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=48, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-24T22:14:38,608 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=49, ppid=48, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-24T22:14:38,608 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37701 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=48 2024-11-24T22:14:38,677 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40087 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=63842f73863e9e7b9587486b122e6aae, server=bd6592c37bb2,40087,1732486423713 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-24T22:14:38,678 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40087 {}] ipc.CallRunner(138): callId: 56 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40202 deadline: 1732486538671, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=63842f73863e9e7b9587486b122e6aae, server=bd6592c37bb2,40087,1732486423713 2024-11-24T22:14:38,678 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40087 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=63842f73863e9e7b9587486b122e6aae, server=bd6592c37bb2,40087,1732486423713 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-24T22:14:38,678 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40087 {}] ipc.CallRunner(138): callId: 140 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40206 deadline: 1732486538671, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=63842f73863e9e7b9587486b122e6aae, server=bd6592c37bb2,40087,1732486423713 2024-11-24T22:14:38,679 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732486460633.63842f73863e9e7b9587486b122e6aae., hostname=bd6592c37bb2,40087,1732486423713, seqNum=5 , the old value is region=TestAcidGuarantees,,1732486460633.63842f73863e9e7b9587486b122e6aae., hostname=bd6592c37bb2,40087,1732486423713, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=63842f73863e9e7b9587486b122e6aae, server=bd6592c37bb2,40087,1732486423713 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T22:14:38,679 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40087 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=63842f73863e9e7b9587486b122e6aae, server=bd6592c37bb2,40087,1732486423713 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-24T22:14:38,679 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732486460633.63842f73863e9e7b9587486b122e6aae., hostname=bd6592c37bb2,40087,1732486423713, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=63842f73863e9e7b9587486b122e6aae, server=bd6592c37bb2,40087,1732486423713 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T22:14:38,679 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732486460633.63842f73863e9e7b9587486b122e6aae., hostname=bd6592c37bb2,40087,1732486423713, seqNum=5 because the exception is null or not the one we care about 2024-11-24T22:14:38,679 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40087 {}] ipc.CallRunner(138): callId: 131 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40244 deadline: 1732486538671, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=63842f73863e9e7b9587486b122e6aae, server=bd6592c37bb2,40087,1732486423713 2024-11-24T22:14:38,679 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732486460633.63842f73863e9e7b9587486b122e6aae., hostname=bd6592c37bb2,40087,1732486423713, seqNum=5 , the old value is region=TestAcidGuarantees,,1732486460633.63842f73863e9e7b9587486b122e6aae., hostname=bd6592c37bb2,40087,1732486423713, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=63842f73863e9e7b9587486b122e6aae, server=bd6592c37bb2,40087,1732486423713 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T22:14:38,679 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732486460633.63842f73863e9e7b9587486b122e6aae., hostname=bd6592c37bb2,40087,1732486423713, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=63842f73863e9e7b9587486b122e6aae, server=bd6592c37bb2,40087,1732486423713 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T22:14:38,679 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732486460633.63842f73863e9e7b9587486b122e6aae., hostname=bd6592c37bb2,40087,1732486423713, seqNum=5 because the exception is null or not the one we care about 2024-11-24T22:14:38,679 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732486460633.63842f73863e9e7b9587486b122e6aae., hostname=bd6592c37bb2,40087,1732486423713, seqNum=5 , the old value is region=TestAcidGuarantees,,1732486460633.63842f73863e9e7b9587486b122e6aae., hostname=bd6592c37bb2,40087,1732486423713, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=63842f73863e9e7b9587486b122e6aae, server=bd6592c37bb2,40087,1732486423713 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T22:14:38,679 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732486460633.63842f73863e9e7b9587486b122e6aae., hostname=bd6592c37bb2,40087,1732486423713, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=63842f73863e9e7b9587486b122e6aae, server=bd6592c37bb2,40087,1732486423713 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T22:14:38,679 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732486460633.63842f73863e9e7b9587486b122e6aae., hostname=bd6592c37bb2,40087,1732486423713, seqNum=5 because the exception is null or not the one we care about 2024-11-24T22:14:38,711 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37701 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=48 2024-11-24T22:14:38,751 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T22:14:38,757 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241124dff701258fc247db9f301f1ffefc5958_63842f73863e9e7b9587486b122e6aae to hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241124dff701258fc247db9f301f1ffefc5958_63842f73863e9e7b9587486b122e6aae 2024-11-24T22:14:38,758 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/63842f73863e9e7b9587486b122e6aae/.tmp/A/4e14986f7371464d829e437f79ceb481, store: [table=TestAcidGuarantees family=A region=63842f73863e9e7b9587486b122e6aae] 2024-11-24T22:14:38,759 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/63842f73863e9e7b9587486b122e6aae/.tmp/A/4e14986f7371464d829e437f79ceb481 is 175, key is test_row_0/A:col10/1732486478332/Put/seqid=0 2024-11-24T22:14:38,764 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=40087 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=49 2024-11-24T22:14:38,764 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bd6592c37bb2:0-1 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732486460633.63842f73863e9e7b9587486b122e6aae. 2024-11-24T22:14:38,764 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bd6592c37bb2:0-1 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732486460633.63842f73863e9e7b9587486b122e6aae. as already flushing 2024-11-24T22:14:38,764 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bd6592c37bb2:0-1 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732486460633.63842f73863e9e7b9587486b122e6aae. 2024-11-24T22:14:38,764 ERROR [RS_FLUSH_OPERATIONS-regionserver/bd6592c37bb2:0-1 {event_type=RS_FLUSH_REGIONS, pid=49}] handler.RSProcedureHandler(58): pid=49 java.io.IOException: Unable to complete flush {ENCODED => 63842f73863e9e7b9587486b122e6aae, NAME => 'TestAcidGuarantees,,1732486460633.63842f73863e9e7b9587486b122e6aae.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T22:14:38,765 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bd6592c37bb2:0-1 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=49 java.io.IOException: Unable to complete flush {ENCODED => 63842f73863e9e7b9587486b122e6aae, NAME => 'TestAcidGuarantees,,1732486460633.63842f73863e9e7b9587486b122e6aae.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T22:14:38,765 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37701 {}] master.HMaster(4177): Remote procedure failed, pid=49 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 63842f73863e9e7b9587486b122e6aae, NAME => 'TestAcidGuarantees,,1732486460633.63842f73863e9e7b9587486b122e6aae.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 63842f73863e9e7b9587486b122e6aae, NAME => 'TestAcidGuarantees,,1732486460633.63842f73863e9e7b9587486b122e6aae.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T22:14:38,791 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45929 is added to blk_1073742016_1192 (size=39949) 2024-11-24T22:14:38,792 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=308, memsize=47.0 K, hasBloomFilter=true, into tmp file hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/63842f73863e9e7b9587486b122e6aae/.tmp/A/4e14986f7371464d829e437f79ceb481 2024-11-24T22:14:38,807 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/63842f73863e9e7b9587486b122e6aae/.tmp/B/a4f2a2e251974d68900f71c211d0d76c is 50, key is test_row_0/B:col10/1732486478332/Put/seqid=0 2024-11-24T22:14:38,839 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45929 is added to blk_1073742017_1193 (size=12301) 2024-11-24T22:14:38,841 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=46.96 KB at sequenceid=308 (bloomFilter=true), to=hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/63842f73863e9e7b9587486b122e6aae/.tmp/B/a4f2a2e251974d68900f71c211d0d76c 2024-11-24T22:14:38,856 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/63842f73863e9e7b9587486b122e6aae/.tmp/C/7fcba68ad79e4eb7b261b7b5aad297e5 is 50, key is test_row_0/C:col10/1732486478332/Put/seqid=0 2024-11-24T22:14:38,900 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45929 is added to blk_1073742018_1194 (size=12301) 2024-11-24T22:14:38,909 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=46.96 KB at sequenceid=308 (bloomFilter=true), to=hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/63842f73863e9e7b9587486b122e6aae/.tmp/C/7fcba68ad79e4eb7b261b7b5aad297e5 2024-11-24T22:14:38,916 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/63842f73863e9e7b9587486b122e6aae/.tmp/A/4e14986f7371464d829e437f79ceb481 as hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/63842f73863e9e7b9587486b122e6aae/A/4e14986f7371464d829e437f79ceb481 2024-11-24T22:14:38,920 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=40087 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=49 2024-11-24T22:14:38,920 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bd6592c37bb2:0-2 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732486460633.63842f73863e9e7b9587486b122e6aae. 2024-11-24T22:14:38,920 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bd6592c37bb2:0-2 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732486460633.63842f73863e9e7b9587486b122e6aae. as already flushing 2024-11-24T22:14:38,921 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bd6592c37bb2:0-2 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732486460633.63842f73863e9e7b9587486b122e6aae. 2024-11-24T22:14:38,921 ERROR [RS_FLUSH_OPERATIONS-regionserver/bd6592c37bb2:0-2 {event_type=RS_FLUSH_REGIONS, pid=49}] handler.RSProcedureHandler(58): pid=49 java.io.IOException: Unable to complete flush {ENCODED => 63842f73863e9e7b9587486b122e6aae, NAME => 'TestAcidGuarantees,,1732486460633.63842f73863e9e7b9587486b122e6aae.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T22:14:38,921 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bd6592c37bb2:0-2 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=49 java.io.IOException: Unable to complete flush {ENCODED => 63842f73863e9e7b9587486b122e6aae, NAME => 'TestAcidGuarantees,,1732486460633.63842f73863e9e7b9587486b122e6aae.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T22:14:38,921 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37701 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=48 2024-11-24T22:14:38,922 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37701 {}] master.HMaster(4177): Remote procedure failed, pid=49 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 63842f73863e9e7b9587486b122e6aae, NAME => 'TestAcidGuarantees,,1732486460633.63842f73863e9e7b9587486b122e6aae.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 63842f73863e9e7b9587486b122e6aae, NAME => 'TestAcidGuarantees,,1732486460633.63842f73863e9e7b9587486b122e6aae.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T22:14:38,927 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/63842f73863e9e7b9587486b122e6aae/A/4e14986f7371464d829e437f79ceb481, entries=200, sequenceid=308, filesize=39.0 K 2024-11-24T22:14:38,929 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/63842f73863e9e7b9587486b122e6aae/.tmp/B/a4f2a2e251974d68900f71c211d0d76c as hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/63842f73863e9e7b9587486b122e6aae/B/a4f2a2e251974d68900f71c211d0d76c 2024-11-24T22:14:38,939 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/63842f73863e9e7b9587486b122e6aae/B/a4f2a2e251974d68900f71c211d0d76c, entries=150, sequenceid=308, filesize=12.0 K 2024-11-24T22:14:38,940 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/63842f73863e9e7b9587486b122e6aae/.tmp/C/7fcba68ad79e4eb7b261b7b5aad297e5 as hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/63842f73863e9e7b9587486b122e6aae/C/7fcba68ad79e4eb7b261b7b5aad297e5 2024-11-24T22:14:38,953 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/63842f73863e9e7b9587486b122e6aae/C/7fcba68ad79e4eb7b261b7b5aad297e5, entries=150, sequenceid=308, filesize=12.0 K 2024-11-24T22:14:38,954 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~140.89 KB/144270, heapSize ~395.48 KB/404976, currentSize=40.25 KB/41220 for 63842f73863e9e7b9587486b122e6aae in 621ms, sequenceid=308, compaction requested=true 2024-11-24T22:14:38,954 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 63842f73863e9e7b9587486b122e6aae: 2024-11-24T22:14:38,955 DEBUG [RS:0;bd6592c37bb2:40087-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-24T22:14:38,955 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 63842f73863e9e7b9587486b122e6aae:A, priority=-2147483648, current under compaction store size is 1 2024-11-24T22:14:38,956 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-24T22:14:38,956 DEBUG [RS:0;bd6592c37bb2:40087-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-24T22:14:38,956 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 63842f73863e9e7b9587486b122e6aae:B, priority=-2147483648, current under compaction store size is 2 2024-11-24T22:14:38,956 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-24T22:14:38,956 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 63842f73863e9e7b9587486b122e6aae:C, priority=-2147483648, current under compaction store size is 3 2024-11-24T22:14:38,956 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-24T22:14:38,958 DEBUG [RS:0;bd6592c37bb2:40087-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 151451 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-24T22:14:38,958 DEBUG [RS:0;bd6592c37bb2:40087-shortCompactions-0 {}] regionserver.HStore(1541): 63842f73863e9e7b9587486b122e6aae/A is initiating minor compaction (all files) 2024-11-24T22:14:38,958 INFO [RS:0;bd6592c37bb2:40087-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 63842f73863e9e7b9587486b122e6aae/A in TestAcidGuarantees,,1732486460633.63842f73863e9e7b9587486b122e6aae. 2024-11-24T22:14:38,958 INFO [RS:0;bd6592c37bb2:40087-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/63842f73863e9e7b9587486b122e6aae/A/cf64aec7f35742de8b3fb00ac564ba08, hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/63842f73863e9e7b9587486b122e6aae/A/ecdf5fa4631f4cc4bfccf8ac391dfc88, hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/63842f73863e9e7b9587486b122e6aae/A/a5c5777a76c04dc7a2b81e81d7e613eb, hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/63842f73863e9e7b9587486b122e6aae/A/4e14986f7371464d829e437f79ceb481] into tmpdir=hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/63842f73863e9e7b9587486b122e6aae/.tmp, totalSize=147.9 K 2024-11-24T22:14:38,958 INFO [RS:0;bd6592c37bb2:40087-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=12 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1732486460633.63842f73863e9e7b9587486b122e6aae. 2024-11-24T22:14:38,958 DEBUG [RS:0;bd6592c37bb2:40087-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1732486460633.63842f73863e9e7b9587486b122e6aae. files: [hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/63842f73863e9e7b9587486b122e6aae/A/cf64aec7f35742de8b3fb00ac564ba08, hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/63842f73863e9e7b9587486b122e6aae/A/ecdf5fa4631f4cc4bfccf8ac391dfc88, hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/63842f73863e9e7b9587486b122e6aae/A/a5c5777a76c04dc7a2b81e81d7e613eb, hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/63842f73863e9e7b9587486b122e6aae/A/4e14986f7371464d829e437f79ceb481] 2024-11-24T22:14:38,959 DEBUG [RS:0;bd6592c37bb2:40087-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 49500 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-24T22:14:38,959 DEBUG [RS:0;bd6592c37bb2:40087-longCompactions-0 {}] regionserver.HStore(1541): 63842f73863e9e7b9587486b122e6aae/B is initiating minor compaction (all files) 2024-11-24T22:14:38,959 INFO [RS:0;bd6592c37bb2:40087-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 63842f73863e9e7b9587486b122e6aae/B in TestAcidGuarantees,,1732486460633.63842f73863e9e7b9587486b122e6aae. 2024-11-24T22:14:38,959 INFO [RS:0;bd6592c37bb2:40087-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/63842f73863e9e7b9587486b122e6aae/B/455dcb4f2aca48b98940fe2ebddff3fb, hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/63842f73863e9e7b9587486b122e6aae/B/81e2cda377e2474c933a28ab8f351261, hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/63842f73863e9e7b9587486b122e6aae/B/09eff7dc8cda475b93df975f70ffcc4a, hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/63842f73863e9e7b9587486b122e6aae/B/a4f2a2e251974d68900f71c211d0d76c] into tmpdir=hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/63842f73863e9e7b9587486b122e6aae/.tmp, totalSize=48.3 K 2024-11-24T22:14:38,960 DEBUG [RS:0;bd6592c37bb2:40087-longCompactions-0 {}] compactions.Compactor(225): Compacting 455dcb4f2aca48b98940fe2ebddff3fb, keycount=150, bloomtype=ROW, size=12.4 K, encoding=NONE, compression=NONE, seqNum=220, earliestPutTs=1732486473479 2024-11-24T22:14:38,960 DEBUG [RS:0;bd6592c37bb2:40087-shortCompactions-0 {}] compactions.Compactor(225): Compacting cf64aec7f35742de8b3fb00ac564ba08, keycount=150, bloomtype=ROW, size=31.0 K, encoding=NONE, compression=NONE, seqNum=220, earliestPutTs=1732486473479 2024-11-24T22:14:38,961 DEBUG [RS:0;bd6592c37bb2:40087-longCompactions-0 {}] compactions.Compactor(225): Compacting 81e2cda377e2474c933a28ab8f351261, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=259, earliestPutTs=1732486476028 2024-11-24T22:14:38,961 DEBUG [RS:0;bd6592c37bb2:40087-shortCompactions-0 {}] compactions.Compactor(225): Compacting ecdf5fa4631f4cc4bfccf8ac391dfc88, keycount=250, bloomtype=ROW, size=47.4 K, encoding=NONE, compression=NONE, seqNum=259, earliestPutTs=1732486476027 2024-11-24T22:14:38,961 DEBUG [RS:0;bd6592c37bb2:40087-longCompactions-0 {}] compactions.Compactor(225): Compacting 09eff7dc8cda475b93df975f70ffcc4a, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=271, earliestPutTs=1732486477206 2024-11-24T22:14:38,962 DEBUG [RS:0;bd6592c37bb2:40087-shortCompactions-0 {}] compactions.Compactor(225): Compacting a5c5777a76c04dc7a2b81e81d7e613eb, keycount=150, bloomtype=ROW, size=30.5 K, encoding=NONE, compression=NONE, seqNum=271, earliestPutTs=1732486477206 2024-11-24T22:14:38,962 DEBUG [RS:0;bd6592c37bb2:40087-longCompactions-0 {}] compactions.Compactor(225): Compacting a4f2a2e251974d68900f71c211d0d76c, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=308, earliestPutTs=1732486477695 2024-11-24T22:14:38,962 DEBUG [RS:0;bd6592c37bb2:40087-shortCompactions-0 {}] compactions.Compactor(225): Compacting 4e14986f7371464d829e437f79ceb481, keycount=200, bloomtype=ROW, size=39.0 K, encoding=NONE, compression=NONE, seqNum=308, earliestPutTs=1732486477678 2024-11-24T22:14:38,984 INFO [RS:0;bd6592c37bb2:40087-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 63842f73863e9e7b9587486b122e6aae#B#compaction#159 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-24T22:14:38,985 DEBUG [RS:0;bd6592c37bb2:40087-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/63842f73863e9e7b9587486b122e6aae/.tmp/B/c239f7036c684af48076a9457f4f24aa is 50, key is test_row_0/B:col10/1732486478332/Put/seqid=0 2024-11-24T22:14:38,987 INFO [RS:0;bd6592c37bb2:40087-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=63842f73863e9e7b9587486b122e6aae] 2024-11-24T22:14:38,999 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40087 {}] regionserver.HRegion(8855): Flush requested on 63842f73863e9e7b9587486b122e6aae 2024-11-24T22:14:39,000 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 63842f73863e9e7b9587486b122e6aae 3/3 column families, dataSize=46.96 KB heapSize=140.20 KB 2024-11-24T22:14:39,001 DEBUG [RS:0;bd6592c37bb2:40087-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e20241124b30b1f6208cd44ee978a2a7670b963ba_63842f73863e9e7b9587486b122e6aae store=[table=TestAcidGuarantees family=A region=63842f73863e9e7b9587486b122e6aae] 2024-11-24T22:14:39,002 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 63842f73863e9e7b9587486b122e6aae, store=A 2024-11-24T22:14:39,002 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-24T22:14:39,002 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 63842f73863e9e7b9587486b122e6aae, store=B 2024-11-24T22:14:39,002 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-24T22:14:39,002 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 63842f73863e9e7b9587486b122e6aae, store=C 2024-11-24T22:14:39,002 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-24T22:14:39,004 DEBUG [RS:0;bd6592c37bb2:40087-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e20241124b30b1f6208cd44ee978a2a7670b963ba_63842f73863e9e7b9587486b122e6aae, store=[table=TestAcidGuarantees family=A region=63842f73863e9e7b9587486b122e6aae] 2024-11-24T22:14:39,004 DEBUG [RS:0;bd6592c37bb2:40087-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241124b30b1f6208cd44ee978a2a7670b963ba_63842f73863e9e7b9587486b122e6aae because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=63842f73863e9e7b9587486b122e6aae] 2024-11-24T22:14:39,029 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411242c097a4926bf4978b6282666bacb85cc_63842f73863e9e7b9587486b122e6aae is 50, key is test_row_0/A:col10/1732486478350/Put/seqid=0 2024-11-24T22:14:39,075 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=40087 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=49 2024-11-24T22:14:39,075 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bd6592c37bb2:0-0 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732486460633.63842f73863e9e7b9587486b122e6aae. 2024-11-24T22:14:39,075 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bd6592c37bb2:0-0 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732486460633.63842f73863e9e7b9587486b122e6aae. as already flushing 2024-11-24T22:14:39,075 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bd6592c37bb2:0-0 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732486460633.63842f73863e9e7b9587486b122e6aae. 2024-11-24T22:14:39,075 ERROR [RS_FLUSH_OPERATIONS-regionserver/bd6592c37bb2:0-0 {event_type=RS_FLUSH_REGIONS, pid=49}] handler.RSProcedureHandler(58): pid=49 java.io.IOException: Unable to complete flush {ENCODED => 63842f73863e9e7b9587486b122e6aae, NAME => 'TestAcidGuarantees,,1732486460633.63842f73863e9e7b9587486b122e6aae.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T22:14:39,076 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bd6592c37bb2:0-0 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=49 java.io.IOException: Unable to complete flush {ENCODED => 63842f73863e9e7b9587486b122e6aae, NAME => 'TestAcidGuarantees,,1732486460633.63842f73863e9e7b9587486b122e6aae.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T22:14:39,076 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37701 {}] master.HMaster(4177): Remote procedure failed, pid=49 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 63842f73863e9e7b9587486b122e6aae, NAME => 'TestAcidGuarantees,,1732486460633.63842f73863e9e7b9587486b122e6aae.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 63842f73863e9e7b9587486b122e6aae, NAME => 'TestAcidGuarantees,,1732486460633.63842f73863e9e7b9587486b122e6aae.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T22:14:39,080 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45929 is added to blk_1073742020_1196 (size=4469) 2024-11-24T22:14:39,081 INFO [RS:0;bd6592c37bb2:40087-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 63842f73863e9e7b9587486b122e6aae#A#compaction#160 average throughput is 0.26 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-24T22:14:39,082 DEBUG [RS:0;bd6592c37bb2:40087-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/63842f73863e9e7b9587486b122e6aae/.tmp/A/e4f431b6491e4a06b5c36163eb80cecf is 175, key is test_row_0/A:col10/1732486478332/Put/seqid=0 2024-11-24T22:14:39,091 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/bd6592c37bb2:0-8 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=209, cellSize=9565, totalHeapSize=21976, min timestamp=1732486479070, max timestamp=1732486479088Num uniques 859; 2024-11-24T22:14:39,091 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/bd6592c37bb2:0-8 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=9565, suffix heap size=117064,new segment heap size=21976 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=209 2024-11-24T22:14:39,091 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/bd6592c37bb2:0-2 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=209, cellSize=9565, totalHeapSize=21976, min timestamp=1732486479070, max timestamp=1732486479088Num uniques 859; 2024-11-24T22:14:39,092 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/bd6592c37bb2:0-2 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=9565, suffix heap size=117064,new segment heap size=21976 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=209 2024-11-24T22:14:39,094 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/bd6592c37bb2:0-3 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=209, cellSize=9565, totalHeapSize=21976, min timestamp=1732486479070, max timestamp=1732486479091Num uniques 859; 2024-11-24T22:14:39,094 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/bd6592c37bb2:0-3 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=9565, suffix heap size=117064,new segment heap size=21976 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=209 2024-11-24T22:14:39,097 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45929 is added to blk_1073742019_1195 (size=12983) 2024-11-24T22:14:39,103 DEBUG [RS:0;bd6592c37bb2:40087-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/63842f73863e9e7b9587486b122e6aae/.tmp/B/c239f7036c684af48076a9457f4f24aa as hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/63842f73863e9e7b9587486b122e6aae/B/c239f7036c684af48076a9457f4f24aa 2024-11-24T22:14:39,112 INFO [RS:0;bd6592c37bb2:40087-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 63842f73863e9e7b9587486b122e6aae/B of 63842f73863e9e7b9587486b122e6aae into c239f7036c684af48076a9457f4f24aa(size=12.7 K), total size for store is 12.7 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-24T22:14:39,112 DEBUG [RS:0;bd6592c37bb2:40087-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 63842f73863e9e7b9587486b122e6aae: 2024-11-24T22:14:39,113 INFO [RS:0;bd6592c37bb2:40087-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732486460633.63842f73863e9e7b9587486b122e6aae., storeName=63842f73863e9e7b9587486b122e6aae/B, priority=12, startTime=1732486478956; duration=0sec 2024-11-24T22:14:39,113 DEBUG [RS:0;bd6592c37bb2:40087-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-24T22:14:39,113 DEBUG [RS:0;bd6592c37bb2:40087-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 63842f73863e9e7b9587486b122e6aae:B 2024-11-24T22:14:39,113 DEBUG [RS:0;bd6592c37bb2:40087-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-24T22:14:39,117 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45929 is added to blk_1073742021_1197 (size=12454) 2024-11-24T22:14:39,124 DEBUG [RS:0;bd6592c37bb2:40087-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 49500 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-24T22:14:39,124 DEBUG [RS:0;bd6592c37bb2:40087-longCompactions-0 {}] regionserver.HStore(1541): 63842f73863e9e7b9587486b122e6aae/C is initiating minor compaction (all files) 2024-11-24T22:14:39,125 INFO [RS:0;bd6592c37bb2:40087-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 63842f73863e9e7b9587486b122e6aae/C in TestAcidGuarantees,,1732486460633.63842f73863e9e7b9587486b122e6aae. 2024-11-24T22:14:39,125 INFO [RS:0;bd6592c37bb2:40087-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/63842f73863e9e7b9587486b122e6aae/C/17307ba8f7ec483e8fc9c7d11b47d475, hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/63842f73863e9e7b9587486b122e6aae/C/c0250f2ef6c147c1ac6f1adc8c0836d6, hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/63842f73863e9e7b9587486b122e6aae/C/253f3fb757454dfcbb4d4a532093bc69, hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/63842f73863e9e7b9587486b122e6aae/C/7fcba68ad79e4eb7b261b7b5aad297e5] into tmpdir=hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/63842f73863e9e7b9587486b122e6aae/.tmp, totalSize=48.3 K 2024-11-24T22:14:39,125 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T22:14:39,125 DEBUG [RS:0;bd6592c37bb2:40087-longCompactions-0 {}] compactions.Compactor(225): Compacting 17307ba8f7ec483e8fc9c7d11b47d475, keycount=150, bloomtype=ROW, size=12.4 K, encoding=NONE, compression=NONE, seqNum=220, earliestPutTs=1732486473479 2024-11-24T22:14:39,127 DEBUG [RS:0;bd6592c37bb2:40087-longCompactions-0 {}] compactions.Compactor(225): Compacting c0250f2ef6c147c1ac6f1adc8c0836d6, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=259, earliestPutTs=1732486476028 2024-11-24T22:14:39,127 DEBUG [RS:0;bd6592c37bb2:40087-longCompactions-0 {}] compactions.Compactor(225): Compacting 253f3fb757454dfcbb4d4a532093bc69, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=271, earliestPutTs=1732486477206 2024-11-24T22:14:39,128 DEBUG [RS:0;bd6592c37bb2:40087-longCompactions-0 {}] compactions.Compactor(225): Compacting 7fcba68ad79e4eb7b261b7b5aad297e5, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=308, earliestPutTs=1732486477695 2024-11-24T22:14:39,130 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411242c097a4926bf4978b6282666bacb85cc_63842f73863e9e7b9587486b122e6aae to hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411242c097a4926bf4978b6282666bacb85cc_63842f73863e9e7b9587486b122e6aae 2024-11-24T22:14:39,132 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/63842f73863e9e7b9587486b122e6aae/.tmp/A/22daac1eb902430cb4be1109cb6cfa80, store: [table=TestAcidGuarantees family=A region=63842f73863e9e7b9587486b122e6aae] 2024-11-24T22:14:39,133 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/63842f73863e9e7b9587486b122e6aae/.tmp/A/22daac1eb902430cb4be1109cb6cfa80 is 175, key is test_row_0/A:col10/1732486478350/Put/seqid=0 2024-11-24T22:14:39,137 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45929 is added to blk_1073742022_1198 (size=31937) 2024-11-24T22:14:39,148 DEBUG [RS:0;bd6592c37bb2:40087-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/63842f73863e9e7b9587486b122e6aae/.tmp/A/e4f431b6491e4a06b5c36163eb80cecf as hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/63842f73863e9e7b9587486b122e6aae/A/e4f431b6491e4a06b5c36163eb80cecf 2024-11-24T22:14:39,158 INFO [RS:0;bd6592c37bb2:40087-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 63842f73863e9e7b9587486b122e6aae/A of 63842f73863e9e7b9587486b122e6aae into e4f431b6491e4a06b5c36163eb80cecf(size=31.2 K), total size for store is 31.2 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-24T22:14:39,158 DEBUG [RS:0;bd6592c37bb2:40087-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 63842f73863e9e7b9587486b122e6aae: 2024-11-24T22:14:39,158 INFO [RS:0;bd6592c37bb2:40087-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732486460633.63842f73863e9e7b9587486b122e6aae., storeName=63842f73863e9e7b9587486b122e6aae/A, priority=12, startTime=1732486478954; duration=0sec 2024-11-24T22:14:39,158 DEBUG [RS:0;bd6592c37bb2:40087-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-24T22:14:39,158 DEBUG [RS:0;bd6592c37bb2:40087-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 63842f73863e9e7b9587486b122e6aae:A 2024-11-24T22:14:39,172 INFO [RS:0;bd6592c37bb2:40087-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 63842f73863e9e7b9587486b122e6aae#C#compaction#162 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-24T22:14:39,172 DEBUG [RS:0;bd6592c37bb2:40087-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/63842f73863e9e7b9587486b122e6aae/.tmp/C/38ae68e5da07426fa5ecf7872ef45fed is 50, key is test_row_0/C:col10/1732486478332/Put/seqid=0 2024-11-24T22:14:39,180 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45929 is added to blk_1073742023_1199 (size=31255) 2024-11-24T22:14:39,180 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=319, memsize=17.9 K, hasBloomFilter=true, into tmp file hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/63842f73863e9e7b9587486b122e6aae/.tmp/A/22daac1eb902430cb4be1109cb6cfa80 2024-11-24T22:14:39,180 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40087 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=63842f73863e9e7b9587486b122e6aae, server=bd6592c37bb2,40087,1732486423713 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-24T22:14:39,181 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40087 {}] ipc.CallRunner(138): callId: 153 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40206 deadline: 1732486539172, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=63842f73863e9e7b9587486b122e6aae, server=bd6592c37bb2,40087,1732486423713 2024-11-24T22:14:39,181 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732486460633.63842f73863e9e7b9587486b122e6aae., hostname=bd6592c37bb2,40087,1732486423713, seqNum=5 , the old value is region=TestAcidGuarantees,,1732486460633.63842f73863e9e7b9587486b122e6aae., hostname=bd6592c37bb2,40087,1732486423713, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=63842f73863e9e7b9587486b122e6aae, server=bd6592c37bb2,40087,1732486423713 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T22:14:39,181 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732486460633.63842f73863e9e7b9587486b122e6aae., hostname=bd6592c37bb2,40087,1732486423713, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=63842f73863e9e7b9587486b122e6aae, server=bd6592c37bb2,40087,1732486423713 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T22:14:39,181 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732486460633.63842f73863e9e7b9587486b122e6aae., hostname=bd6592c37bb2,40087,1732486423713, seqNum=5 because the exception is null or not the one we care about 2024-11-24T22:14:39,191 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/63842f73863e9e7b9587486b122e6aae/.tmp/B/e2d876c1c5b849819f4edeeba50d9ad3 is 50, key is test_row_0/B:col10/1732486478350/Put/seqid=0 2024-11-24T22:14:39,196 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40087 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=63842f73863e9e7b9587486b122e6aae, server=bd6592c37bb2,40087,1732486423713 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-24T22:14:39,196 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40087 {}] ipc.CallRunner(138): callId: 143 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40244 deadline: 1732486539176, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=63842f73863e9e7b9587486b122e6aae, server=bd6592c37bb2,40087,1732486423713 2024-11-24T22:14:39,196 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40087 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=63842f73863e9e7b9587486b122e6aae, server=bd6592c37bb2,40087,1732486423713 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-24T22:14:39,196 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40087 {}] ipc.CallRunner(138): callId: 68 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40202 deadline: 1732486539180, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=63842f73863e9e7b9587486b122e6aae, server=bd6592c37bb2,40087,1732486423713 2024-11-24T22:14:39,197 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732486460633.63842f73863e9e7b9587486b122e6aae., hostname=bd6592c37bb2,40087,1732486423713, seqNum=5 , the old value is region=TestAcidGuarantees,,1732486460633.63842f73863e9e7b9587486b122e6aae., hostname=bd6592c37bb2,40087,1732486423713, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=63842f73863e9e7b9587486b122e6aae, server=bd6592c37bb2,40087,1732486423713 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T22:14:39,197 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732486460633.63842f73863e9e7b9587486b122e6aae., hostname=bd6592c37bb2,40087,1732486423713, seqNum=5 , the old value is region=TestAcidGuarantees,,1732486460633.63842f73863e9e7b9587486b122e6aae., hostname=bd6592c37bb2,40087,1732486423713, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=63842f73863e9e7b9587486b122e6aae, server=bd6592c37bb2,40087,1732486423713 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T22:14:39,197 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732486460633.63842f73863e9e7b9587486b122e6aae., hostname=bd6592c37bb2,40087,1732486423713, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=63842f73863e9e7b9587486b122e6aae, server=bd6592c37bb2,40087,1732486423713 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T22:14:39,197 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732486460633.63842f73863e9e7b9587486b122e6aae., hostname=bd6592c37bb2,40087,1732486423713, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=63842f73863e9e7b9587486b122e6aae, server=bd6592c37bb2,40087,1732486423713 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T22:14:39,197 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732486460633.63842f73863e9e7b9587486b122e6aae., hostname=bd6592c37bb2,40087,1732486423713, seqNum=5 because the exception is null or not the one we care about 2024-11-24T22:14:39,197 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732486460633.63842f73863e9e7b9587486b122e6aae., hostname=bd6592c37bb2,40087,1732486423713, seqNum=5 because the exception is null or not the one we care about 2024-11-24T22:14:39,199 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45929 is added to blk_1073742025_1201 (size=12301) 2024-11-24T22:14:39,201 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=319 (bloomFilter=true), to=hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/63842f73863e9e7b9587486b122e6aae/.tmp/B/e2d876c1c5b849819f4edeeba50d9ad3 2024-11-24T22:14:39,224 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45929 is added to blk_1073742024_1200 (size=12983) 2024-11-24T22:14:39,230 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/63842f73863e9e7b9587486b122e6aae/.tmp/C/c9bc933b9cc242a4a268f760548c19e2 is 50, key is test_row_0/C:col10/1732486478350/Put/seqid=0 2024-11-24T22:14:39,231 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37701 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=48 2024-11-24T22:14:39,231 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=40087 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=49 2024-11-24T22:14:39,231 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bd6592c37bb2:0-1 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732486460633.63842f73863e9e7b9587486b122e6aae. 2024-11-24T22:14:39,232 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bd6592c37bb2:0-1 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732486460633.63842f73863e9e7b9587486b122e6aae. as already flushing 2024-11-24T22:14:39,232 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bd6592c37bb2:0-1 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732486460633.63842f73863e9e7b9587486b122e6aae. 2024-11-24T22:14:39,232 ERROR [RS_FLUSH_OPERATIONS-regionserver/bd6592c37bb2:0-1 {event_type=RS_FLUSH_REGIONS, pid=49}] handler.RSProcedureHandler(58): pid=49 java.io.IOException: Unable to complete flush {ENCODED => 63842f73863e9e7b9587486b122e6aae, NAME => 'TestAcidGuarantees,,1732486460633.63842f73863e9e7b9587486b122e6aae.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T22:14:39,232 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bd6592c37bb2:0-1 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=49 java.io.IOException: Unable to complete flush {ENCODED => 63842f73863e9e7b9587486b122e6aae, NAME => 'TestAcidGuarantees,,1732486460633.63842f73863e9e7b9587486b122e6aae.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T22:14:39,233 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37701 {}] master.HMaster(4177): Remote procedure failed, pid=49 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 63842f73863e9e7b9587486b122e6aae, NAME => 'TestAcidGuarantees,,1732486460633.63842f73863e9e7b9587486b122e6aae.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 63842f73863e9e7b9587486b122e6aae, NAME => 'TestAcidGuarantees,,1732486460633.63842f73863e9e7b9587486b122e6aae.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T22:14:39,253 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45929 is added to blk_1073742026_1202 (size=12301) 2024-11-24T22:14:39,254 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=319 (bloomFilter=true), to=hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/63842f73863e9e7b9587486b122e6aae/.tmp/C/c9bc933b9cc242a4a268f760548c19e2 2024-11-24T22:14:39,260 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/63842f73863e9e7b9587486b122e6aae/.tmp/A/22daac1eb902430cb4be1109cb6cfa80 as hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/63842f73863e9e7b9587486b122e6aae/A/22daac1eb902430cb4be1109cb6cfa80 2024-11-24T22:14:39,266 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/63842f73863e9e7b9587486b122e6aae/A/22daac1eb902430cb4be1109cb6cfa80, entries=150, sequenceid=319, filesize=30.5 K 2024-11-24T22:14:39,267 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/63842f73863e9e7b9587486b122e6aae/.tmp/B/e2d876c1c5b849819f4edeeba50d9ad3 as hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/63842f73863e9e7b9587486b122e6aae/B/e2d876c1c5b849819f4edeeba50d9ad3 2024-11-24T22:14:39,272 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/63842f73863e9e7b9587486b122e6aae/B/e2d876c1c5b849819f4edeeba50d9ad3, entries=150, sequenceid=319, filesize=12.0 K 2024-11-24T22:14:39,274 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/63842f73863e9e7b9587486b122e6aae/.tmp/C/c9bc933b9cc242a4a268f760548c19e2 as hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/63842f73863e9e7b9587486b122e6aae/C/c9bc933b9cc242a4a268f760548c19e2 2024-11-24T22:14:39,282 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/63842f73863e9e7b9587486b122e6aae/C/c9bc933b9cc242a4a268f760548c19e2, entries=150, sequenceid=319, filesize=12.0 K 2024-11-24T22:14:39,287 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~53.67 KB/54960, heapSize ~160.08 KB/163920, currentSize=127.47 KB/130530 for 63842f73863e9e7b9587486b122e6aae in 287ms, sequenceid=319, compaction requested=false 2024-11-24T22:14:39,288 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 63842f73863e9e7b9587486b122e6aae: 2024-11-24T22:14:39,295 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40087 {}] regionserver.HRegion(8855): Flush requested on 63842f73863e9e7b9587486b122e6aae 2024-11-24T22:14:39,295 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 63842f73863e9e7b9587486b122e6aae 3/3 column families, dataSize=134.18 KB heapSize=380.30 KB 2024-11-24T22:14:39,295 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 63842f73863e9e7b9587486b122e6aae, store=A 2024-11-24T22:14:39,295 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-24T22:14:39,295 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 63842f73863e9e7b9587486b122e6aae, store=B 2024-11-24T22:14:39,295 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-24T22:14:39,295 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 63842f73863e9e7b9587486b122e6aae, store=C 2024-11-24T22:14:39,296 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-24T22:14:39,309 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411249831828caf7f4782b0bc8a54ad20c973_63842f73863e9e7b9587486b122e6aae is 50, key is test_row_0/A:col10/1732486479160/Put/seqid=0 2024-11-24T22:14:39,313 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45929 is added to blk_1073742027_1203 (size=14994) 2024-11-24T22:14:39,317 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40087 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=63842f73863e9e7b9587486b122e6aae, server=bd6592c37bb2,40087,1732486423713 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-24T22:14:39,317 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40087 {}] ipc.CallRunner(138): callId: 71 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40202 deadline: 1732486539311, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=63842f73863e9e7b9587486b122e6aae, server=bd6592c37bb2,40087,1732486423713 2024-11-24T22:14:39,317 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40087 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=63842f73863e9e7b9587486b122e6aae, server=bd6592c37bb2,40087,1732486423713 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-24T22:14:39,317 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40087 {}] ipc.CallRunner(138): callId: 146 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40244 deadline: 1732486539312, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=63842f73863e9e7b9587486b122e6aae, server=bd6592c37bb2,40087,1732486423713 2024-11-24T22:14:39,318 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732486460633.63842f73863e9e7b9587486b122e6aae., hostname=bd6592c37bb2,40087,1732486423713, seqNum=5 , the old value is region=TestAcidGuarantees,,1732486460633.63842f73863e9e7b9587486b122e6aae., hostname=bd6592c37bb2,40087,1732486423713, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=63842f73863e9e7b9587486b122e6aae, server=bd6592c37bb2,40087,1732486423713 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T22:14:39,318 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732486460633.63842f73863e9e7b9587486b122e6aae., hostname=bd6592c37bb2,40087,1732486423713, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=63842f73863e9e7b9587486b122e6aae, server=bd6592c37bb2,40087,1732486423713 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T22:14:39,318 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732486460633.63842f73863e9e7b9587486b122e6aae., hostname=bd6592c37bb2,40087,1732486423713, seqNum=5 because the exception is null or not the one we care about 2024-11-24T22:14:39,318 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732486460633.63842f73863e9e7b9587486b122e6aae., hostname=bd6592c37bb2,40087,1732486423713, seqNum=5 , the old value is region=TestAcidGuarantees,,1732486460633.63842f73863e9e7b9587486b122e6aae., hostname=bd6592c37bb2,40087,1732486423713, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=63842f73863e9e7b9587486b122e6aae, server=bd6592c37bb2,40087,1732486423713 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T22:14:39,318 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732486460633.63842f73863e9e7b9587486b122e6aae., hostname=bd6592c37bb2,40087,1732486423713, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=63842f73863e9e7b9587486b122e6aae, server=bd6592c37bb2,40087,1732486423713 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T22:14:39,318 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732486460633.63842f73863e9e7b9587486b122e6aae., hostname=bd6592c37bb2,40087,1732486423713, seqNum=5 because the exception is null or not the one we care about 2024-11-24T22:14:39,319 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40087 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=63842f73863e9e7b9587486b122e6aae, server=bd6592c37bb2,40087,1732486423713 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-24T22:14:39,319 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40087 {}] ipc.CallRunner(138): callId: 158 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40206 deadline: 1732486539316, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=63842f73863e9e7b9587486b122e6aae, server=bd6592c37bb2,40087,1732486423713 2024-11-24T22:14:39,319 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732486460633.63842f73863e9e7b9587486b122e6aae., hostname=bd6592c37bb2,40087,1732486423713, seqNum=5 , the old value is region=TestAcidGuarantees,,1732486460633.63842f73863e9e7b9587486b122e6aae., hostname=bd6592c37bb2,40087,1732486423713, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=63842f73863e9e7b9587486b122e6aae, server=bd6592c37bb2,40087,1732486423713 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T22:14:39,319 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732486460633.63842f73863e9e7b9587486b122e6aae., hostname=bd6592c37bb2,40087,1732486423713, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=63842f73863e9e7b9587486b122e6aae, server=bd6592c37bb2,40087,1732486423713 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T22:14:39,319 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732486460633.63842f73863e9e7b9587486b122e6aae., hostname=bd6592c37bb2,40087,1732486423713, seqNum=5 because the exception is null or not the one we care about 2024-11-24T22:14:39,385 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=40087 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=49 2024-11-24T22:14:39,385 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bd6592c37bb2:0-2 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732486460633.63842f73863e9e7b9587486b122e6aae. 2024-11-24T22:14:39,385 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bd6592c37bb2:0-2 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732486460633.63842f73863e9e7b9587486b122e6aae. as already flushing 2024-11-24T22:14:39,385 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bd6592c37bb2:0-2 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732486460633.63842f73863e9e7b9587486b122e6aae. 2024-11-24T22:14:39,385 ERROR [RS_FLUSH_OPERATIONS-regionserver/bd6592c37bb2:0-2 {event_type=RS_FLUSH_REGIONS, pid=49}] handler.RSProcedureHandler(58): pid=49 java.io.IOException: Unable to complete flush {ENCODED => 63842f73863e9e7b9587486b122e6aae, NAME => 'TestAcidGuarantees,,1732486460633.63842f73863e9e7b9587486b122e6aae.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T22:14:39,385 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bd6592c37bb2:0-2 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=49 java.io.IOException: Unable to complete flush {ENCODED => 63842f73863e9e7b9587486b122e6aae, NAME => 'TestAcidGuarantees,,1732486460633.63842f73863e9e7b9587486b122e6aae.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T22:14:39,386 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37701 {}] master.HMaster(4177): Remote procedure failed, pid=49 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 63842f73863e9e7b9587486b122e6aae, NAME => 'TestAcidGuarantees,,1732486460633.63842f73863e9e7b9587486b122e6aae.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 63842f73863e9e7b9587486b122e6aae, NAME => 'TestAcidGuarantees,,1732486460633.63842f73863e9e7b9587486b122e6aae.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T22:14:39,422 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40087 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=63842f73863e9e7b9587486b122e6aae, server=bd6592c37bb2,40087,1732486423713 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-24T22:14:39,422 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40087 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=63842f73863e9e7b9587486b122e6aae, server=bd6592c37bb2,40087,1732486423713 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-24T22:14:39,422 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40087 {}] ipc.CallRunner(138): callId: 72 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40202 deadline: 1732486539421, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=63842f73863e9e7b9587486b122e6aae, server=bd6592c37bb2,40087,1732486423713 2024-11-24T22:14:39,422 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40087 {}] ipc.CallRunner(138): callId: 147 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40244 deadline: 1732486539421, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=63842f73863e9e7b9587486b122e6aae, server=bd6592c37bb2,40087,1732486423713 2024-11-24T22:14:39,422 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732486460633.63842f73863e9e7b9587486b122e6aae., hostname=bd6592c37bb2,40087,1732486423713, seqNum=5 , the old value is region=TestAcidGuarantees,,1732486460633.63842f73863e9e7b9587486b122e6aae., hostname=bd6592c37bb2,40087,1732486423713, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=63842f73863e9e7b9587486b122e6aae, server=bd6592c37bb2,40087,1732486423713 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T22:14:39,422 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732486460633.63842f73863e9e7b9587486b122e6aae., hostname=bd6592c37bb2,40087,1732486423713, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=63842f73863e9e7b9587486b122e6aae, server=bd6592c37bb2,40087,1732486423713 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T22:14:39,422 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732486460633.63842f73863e9e7b9587486b122e6aae., hostname=bd6592c37bb2,40087,1732486423713, seqNum=5 because the exception is null or not the one we care about 2024-11-24T22:14:39,423 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732486460633.63842f73863e9e7b9587486b122e6aae., hostname=bd6592c37bb2,40087,1732486423713, seqNum=5 , the old value is region=TestAcidGuarantees,,1732486460633.63842f73863e9e7b9587486b122e6aae., hostname=bd6592c37bb2,40087,1732486423713, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=63842f73863e9e7b9587486b122e6aae, server=bd6592c37bb2,40087,1732486423713 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T22:14:39,423 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732486460633.63842f73863e9e7b9587486b122e6aae., hostname=bd6592c37bb2,40087,1732486423713, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=63842f73863e9e7b9587486b122e6aae, server=bd6592c37bb2,40087,1732486423713 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T22:14:39,423 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732486460633.63842f73863e9e7b9587486b122e6aae., hostname=bd6592c37bb2,40087,1732486423713, seqNum=5 because the exception is null or not the one we care about 2024-11-24T22:14:39,436 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40087 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=63842f73863e9e7b9587486b122e6aae, server=bd6592c37bb2,40087,1732486423713 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-24T22:14:39,436 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40087 {}] ipc.CallRunner(138): callId: 159 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40206 deadline: 1732486539430, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=63842f73863e9e7b9587486b122e6aae, server=bd6592c37bb2,40087,1732486423713 2024-11-24T22:14:39,437 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732486460633.63842f73863e9e7b9587486b122e6aae., hostname=bd6592c37bb2,40087,1732486423713, seqNum=5 , the old value is region=TestAcidGuarantees,,1732486460633.63842f73863e9e7b9587486b122e6aae., hostname=bd6592c37bb2,40087,1732486423713, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=63842f73863e9e7b9587486b122e6aae, server=bd6592c37bb2,40087,1732486423713 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T22:14:39,437 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732486460633.63842f73863e9e7b9587486b122e6aae., hostname=bd6592c37bb2,40087,1732486423713, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=63842f73863e9e7b9587486b122e6aae, server=bd6592c37bb2,40087,1732486423713 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T22:14:39,437 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732486460633.63842f73863e9e7b9587486b122e6aae., hostname=bd6592c37bb2,40087,1732486423713, seqNum=5 because the exception is null or not the one we care about 2024-11-24T22:14:39,538 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=40087 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=49 2024-11-24T22:14:39,538 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bd6592c37bb2:0-0 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732486460633.63842f73863e9e7b9587486b122e6aae. 2024-11-24T22:14:39,538 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bd6592c37bb2:0-0 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732486460633.63842f73863e9e7b9587486b122e6aae. as already flushing 2024-11-24T22:14:39,538 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bd6592c37bb2:0-0 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732486460633.63842f73863e9e7b9587486b122e6aae. 2024-11-24T22:14:39,538 ERROR [RS_FLUSH_OPERATIONS-regionserver/bd6592c37bb2:0-0 {event_type=RS_FLUSH_REGIONS, pid=49}] handler.RSProcedureHandler(58): pid=49 java.io.IOException: Unable to complete flush {ENCODED => 63842f73863e9e7b9587486b122e6aae, NAME => 'TestAcidGuarantees,,1732486460633.63842f73863e9e7b9587486b122e6aae.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T22:14:39,538 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bd6592c37bb2:0-0 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=49 java.io.IOException: Unable to complete flush {ENCODED => 63842f73863e9e7b9587486b122e6aae, NAME => 'TestAcidGuarantees,,1732486460633.63842f73863e9e7b9587486b122e6aae.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T22:14:39,540 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37701 {}] master.HMaster(4177): Remote procedure failed, pid=49 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 63842f73863e9e7b9587486b122e6aae, NAME => 'TestAcidGuarantees,,1732486460633.63842f73863e9e7b9587486b122e6aae.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 63842f73863e9e7b9587486b122e6aae, NAME => 'TestAcidGuarantees,,1732486460633.63842f73863e9e7b9587486b122e6aae.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T22:14:39,631 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40087 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=63842f73863e9e7b9587486b122e6aae, server=bd6592c37bb2,40087,1732486423713 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-24T22:14:39,632 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40087 {}] ipc.CallRunner(138): callId: 148 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40244 deadline: 1732486539630, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=63842f73863e9e7b9587486b122e6aae, server=bd6592c37bb2,40087,1732486423713 2024-11-24T22:14:39,632 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732486460633.63842f73863e9e7b9587486b122e6aae., hostname=bd6592c37bb2,40087,1732486423713, seqNum=5 , the old value is region=TestAcidGuarantees,,1732486460633.63842f73863e9e7b9587486b122e6aae., hostname=bd6592c37bb2,40087,1732486423713, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=63842f73863e9e7b9587486b122e6aae, server=bd6592c37bb2,40087,1732486423713 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T22:14:39,632 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732486460633.63842f73863e9e7b9587486b122e6aae., hostname=bd6592c37bb2,40087,1732486423713, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=63842f73863e9e7b9587486b122e6aae, server=bd6592c37bb2,40087,1732486423713 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T22:14:39,632 DEBUG [RS:0;bd6592c37bb2:40087-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/63842f73863e9e7b9587486b122e6aae/.tmp/C/38ae68e5da07426fa5ecf7872ef45fed as hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/63842f73863e9e7b9587486b122e6aae/C/38ae68e5da07426fa5ecf7872ef45fed 2024-11-24T22:14:39,632 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732486460633.63842f73863e9e7b9587486b122e6aae., hostname=bd6592c37bb2,40087,1732486423713, seqNum=5 because the exception is null or not the one we care about 2024-11-24T22:14:39,638 INFO [RS:0;bd6592c37bb2:40087-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 63842f73863e9e7b9587486b122e6aae/C of 63842f73863e9e7b9587486b122e6aae into 38ae68e5da07426fa5ecf7872ef45fed(size=12.7 K), total size for store is 24.7 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-24T22:14:39,638 DEBUG [RS:0;bd6592c37bb2:40087-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 63842f73863e9e7b9587486b122e6aae: 2024-11-24T22:14:39,638 INFO [RS:0;bd6592c37bb2:40087-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732486460633.63842f73863e9e7b9587486b122e6aae., storeName=63842f73863e9e7b9587486b122e6aae/C, priority=12, startTime=1732486478956; duration=0sec 2024-11-24T22:14:39,638 DEBUG [RS:0;bd6592c37bb2:40087-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-24T22:14:39,638 DEBUG [RS:0;bd6592c37bb2:40087-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 63842f73863e9e7b9587486b122e6aae:C 2024-11-24T22:14:39,639 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40087 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=63842f73863e9e7b9587486b122e6aae, server=bd6592c37bb2,40087,1732486423713 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-24T22:14:39,639 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40087 {}] ipc.CallRunner(138): callId: 73 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40202 deadline: 1732486539631, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=63842f73863e9e7b9587486b122e6aae, server=bd6592c37bb2,40087,1732486423713 2024-11-24T22:14:39,640 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732486460633.63842f73863e9e7b9587486b122e6aae., hostname=bd6592c37bb2,40087,1732486423713, seqNum=5 , the old value is region=TestAcidGuarantees,,1732486460633.63842f73863e9e7b9587486b122e6aae., hostname=bd6592c37bb2,40087,1732486423713, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=63842f73863e9e7b9587486b122e6aae, server=bd6592c37bb2,40087,1732486423713 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T22:14:39,640 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732486460633.63842f73863e9e7b9587486b122e6aae., hostname=bd6592c37bb2,40087,1732486423713, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=63842f73863e9e7b9587486b122e6aae, server=bd6592c37bb2,40087,1732486423713 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T22:14:39,640 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732486460633.63842f73863e9e7b9587486b122e6aae., hostname=bd6592c37bb2,40087,1732486423713, seqNum=5 because the exception is null or not the one we care about 2024-11-24T22:14:39,646 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40087 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=63842f73863e9e7b9587486b122e6aae, server=bd6592c37bb2,40087,1732486423713 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-24T22:14:39,646 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40087 {}] ipc.CallRunner(138): callId: 160 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40206 deadline: 1732486539641, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=63842f73863e9e7b9587486b122e6aae, server=bd6592c37bb2,40087,1732486423713 2024-11-24T22:14:39,646 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732486460633.63842f73863e9e7b9587486b122e6aae., hostname=bd6592c37bb2,40087,1732486423713, seqNum=5 , the old value is region=TestAcidGuarantees,,1732486460633.63842f73863e9e7b9587486b122e6aae., hostname=bd6592c37bb2,40087,1732486423713, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=63842f73863e9e7b9587486b122e6aae, server=bd6592c37bb2,40087,1732486423713 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T22:14:39,646 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732486460633.63842f73863e9e7b9587486b122e6aae., hostname=bd6592c37bb2,40087,1732486423713, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=63842f73863e9e7b9587486b122e6aae, server=bd6592c37bb2,40087,1732486423713 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T22:14:39,646 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732486460633.63842f73863e9e7b9587486b122e6aae., hostname=bd6592c37bb2,40087,1732486423713, seqNum=5 because the exception is null or not the one we care about 2024-11-24T22:14:39,694 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=40087 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=49 2024-11-24T22:14:39,694 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bd6592c37bb2:0-1 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732486460633.63842f73863e9e7b9587486b122e6aae. 2024-11-24T22:14:39,694 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bd6592c37bb2:0-1 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732486460633.63842f73863e9e7b9587486b122e6aae. as already flushing 2024-11-24T22:14:39,694 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bd6592c37bb2:0-1 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732486460633.63842f73863e9e7b9587486b122e6aae. 2024-11-24T22:14:39,694 ERROR [RS_FLUSH_OPERATIONS-regionserver/bd6592c37bb2:0-1 {event_type=RS_FLUSH_REGIONS, pid=49}] handler.RSProcedureHandler(58): pid=49 java.io.IOException: Unable to complete flush {ENCODED => 63842f73863e9e7b9587486b122e6aae, NAME => 'TestAcidGuarantees,,1732486460633.63842f73863e9e7b9587486b122e6aae.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T22:14:39,694 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bd6592c37bb2:0-1 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=49 java.io.IOException: Unable to complete flush {ENCODED => 63842f73863e9e7b9587486b122e6aae, NAME => 'TestAcidGuarantees,,1732486460633.63842f73863e9e7b9587486b122e6aae.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T22:14:39,695 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37701 {}] master.HMaster(4177): Remote procedure failed, pid=49 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 63842f73863e9e7b9587486b122e6aae, NAME => 'TestAcidGuarantees,,1732486460633.63842f73863e9e7b9587486b122e6aae.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 63842f73863e9e7b9587486b122e6aae, NAME => 'TestAcidGuarantees,,1732486460633.63842f73863e9e7b9587486b122e6aae.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T22:14:39,714 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T22:14:39,718 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411249831828caf7f4782b0bc8a54ad20c973_63842f73863e9e7b9587486b122e6aae to hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411249831828caf7f4782b0bc8a54ad20c973_63842f73863e9e7b9587486b122e6aae 2024-11-24T22:14:39,719 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/63842f73863e9e7b9587486b122e6aae/.tmp/A/a09a0cf5571f473d87794d3878c6555e, store: [table=TestAcidGuarantees family=A region=63842f73863e9e7b9587486b122e6aae] 2024-11-24T22:14:39,720 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/63842f73863e9e7b9587486b122e6aae/.tmp/A/a09a0cf5571f473d87794d3878c6555e is 175, key is test_row_0/A:col10/1732486479160/Put/seqid=0 2024-11-24T22:14:39,724 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45929 is added to blk_1073742028_1204 (size=39949) 2024-11-24T22:14:39,725 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=357, memsize=44.7 K, hasBloomFilter=true, into tmp file hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/63842f73863e9e7b9587486b122e6aae/.tmp/A/a09a0cf5571f473d87794d3878c6555e 2024-11-24T22:14:39,735 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/63842f73863e9e7b9587486b122e6aae/.tmp/B/053a596b3ce94469af401422641d7f60 is 50, key is test_row_0/B:col10/1732486479160/Put/seqid=0 2024-11-24T22:14:39,739 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45929 is added to blk_1073742029_1205 (size=12301) 2024-11-24T22:14:39,741 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37701 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=48 2024-11-24T22:14:39,847 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=40087 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=49 2024-11-24T22:14:39,848 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bd6592c37bb2:0-2 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732486460633.63842f73863e9e7b9587486b122e6aae. 2024-11-24T22:14:39,848 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bd6592c37bb2:0-2 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732486460633.63842f73863e9e7b9587486b122e6aae. as already flushing 2024-11-24T22:14:39,848 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bd6592c37bb2:0-2 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732486460633.63842f73863e9e7b9587486b122e6aae. 2024-11-24T22:14:39,848 ERROR [RS_FLUSH_OPERATIONS-regionserver/bd6592c37bb2:0-2 {event_type=RS_FLUSH_REGIONS, pid=49}] handler.RSProcedureHandler(58): pid=49 java.io.IOException: Unable to complete flush {ENCODED => 63842f73863e9e7b9587486b122e6aae, NAME => 'TestAcidGuarantees,,1732486460633.63842f73863e9e7b9587486b122e6aae.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T22:14:39,848 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bd6592c37bb2:0-2 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=49 java.io.IOException: Unable to complete flush {ENCODED => 63842f73863e9e7b9587486b122e6aae, NAME => 'TestAcidGuarantees,,1732486460633.63842f73863e9e7b9587486b122e6aae.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T22:14:39,849 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37701 {}] master.HMaster(4177): Remote procedure failed, pid=49 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 63842f73863e9e7b9587486b122e6aae, NAME => 'TestAcidGuarantees,,1732486460633.63842f73863e9e7b9587486b122e6aae.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 63842f73863e9e7b9587486b122e6aae, NAME => 'TestAcidGuarantees,,1732486460633.63842f73863e9e7b9587486b122e6aae.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T22:14:39,951 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40087 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=63842f73863e9e7b9587486b122e6aae, server=bd6592c37bb2,40087,1732486423713 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-24T22:14:39,951 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40087 {}] ipc.CallRunner(138): callId: 149 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40244 deadline: 1732486539941, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=63842f73863e9e7b9587486b122e6aae, server=bd6592c37bb2,40087,1732486423713 2024-11-24T22:14:39,952 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732486460633.63842f73863e9e7b9587486b122e6aae., hostname=bd6592c37bb2,40087,1732486423713, seqNum=5 , the old value is region=TestAcidGuarantees,,1732486460633.63842f73863e9e7b9587486b122e6aae., hostname=bd6592c37bb2,40087,1732486423713, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=63842f73863e9e7b9587486b122e6aae, server=bd6592c37bb2,40087,1732486423713 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T22:14:39,952 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732486460633.63842f73863e9e7b9587486b122e6aae., hostname=bd6592c37bb2,40087,1732486423713, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=63842f73863e9e7b9587486b122e6aae, server=bd6592c37bb2,40087,1732486423713 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T22:14:39,952 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732486460633.63842f73863e9e7b9587486b122e6aae., hostname=bd6592c37bb2,40087,1732486423713, seqNum=5 because the exception is null or not the one we care about 2024-11-24T22:14:39,955 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40087 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=63842f73863e9e7b9587486b122e6aae, server=bd6592c37bb2,40087,1732486423713 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-24T22:14:39,956 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40087 {}] ipc.CallRunner(138): callId: 161 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40206 deadline: 1732486539951, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=63842f73863e9e7b9587486b122e6aae, server=bd6592c37bb2,40087,1732486423713 2024-11-24T22:14:39,956 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732486460633.63842f73863e9e7b9587486b122e6aae., hostname=bd6592c37bb2,40087,1732486423713, seqNum=5 , the old value is region=TestAcidGuarantees,,1732486460633.63842f73863e9e7b9587486b122e6aae., hostname=bd6592c37bb2,40087,1732486423713, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=63842f73863e9e7b9587486b122e6aae, server=bd6592c37bb2,40087,1732486423713 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T22:14:39,956 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732486460633.63842f73863e9e7b9587486b122e6aae., hostname=bd6592c37bb2,40087,1732486423713, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=63842f73863e9e7b9587486b122e6aae, server=bd6592c37bb2,40087,1732486423713 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T22:14:39,956 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732486460633.63842f73863e9e7b9587486b122e6aae., hostname=bd6592c37bb2,40087,1732486423713, seqNum=5 because the exception is null or not the one we care about 2024-11-24T22:14:39,956 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40087 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=63842f73863e9e7b9587486b122e6aae, server=bd6592c37bb2,40087,1732486423713 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-24T22:14:39,956 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40087 {}] ipc.CallRunner(138): callId: 74 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40202 deadline: 1732486539951, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=63842f73863e9e7b9587486b122e6aae, server=bd6592c37bb2,40087,1732486423713 2024-11-24T22:14:39,957 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732486460633.63842f73863e9e7b9587486b122e6aae., hostname=bd6592c37bb2,40087,1732486423713, seqNum=5 , the old value is region=TestAcidGuarantees,,1732486460633.63842f73863e9e7b9587486b122e6aae., hostname=bd6592c37bb2,40087,1732486423713, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=63842f73863e9e7b9587486b122e6aae, server=bd6592c37bb2,40087,1732486423713 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T22:14:39,957 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732486460633.63842f73863e9e7b9587486b122e6aae., hostname=bd6592c37bb2,40087,1732486423713, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=63842f73863e9e7b9587486b122e6aae, server=bd6592c37bb2,40087,1732486423713 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T22:14:39,957 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732486460633.63842f73863e9e7b9587486b122e6aae., hostname=bd6592c37bb2,40087,1732486423713, seqNum=5 because the exception is null or not the one we care about 2024-11-24T22:14:40,001 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=40087 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=49 2024-11-24T22:14:40,001 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bd6592c37bb2:0-0 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732486460633.63842f73863e9e7b9587486b122e6aae. 2024-11-24T22:14:40,001 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bd6592c37bb2:0-0 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732486460633.63842f73863e9e7b9587486b122e6aae. as already flushing 2024-11-24T22:14:40,001 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bd6592c37bb2:0-0 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732486460633.63842f73863e9e7b9587486b122e6aae. 2024-11-24T22:14:40,001 ERROR [RS_FLUSH_OPERATIONS-regionserver/bd6592c37bb2:0-0 {event_type=RS_FLUSH_REGIONS, pid=49}] handler.RSProcedureHandler(58): pid=49 java.io.IOException: Unable to complete flush {ENCODED => 63842f73863e9e7b9587486b122e6aae, NAME => 'TestAcidGuarantees,,1732486460633.63842f73863e9e7b9587486b122e6aae.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T22:14:40,002 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bd6592c37bb2:0-0 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=49 java.io.IOException: Unable to complete flush {ENCODED => 63842f73863e9e7b9587486b122e6aae, NAME => 'TestAcidGuarantees,,1732486460633.63842f73863e9e7b9587486b122e6aae.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T22:14:40,002 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37701 {}] master.HMaster(4177): Remote procedure failed, pid=49 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 63842f73863e9e7b9587486b122e6aae, NAME => 'TestAcidGuarantees,,1732486460633.63842f73863e9e7b9587486b122e6aae.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 63842f73863e9e7b9587486b122e6aae, NAME => 'TestAcidGuarantees,,1732486460633.63842f73863e9e7b9587486b122e6aae.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T22:14:40,140 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=357 (bloomFilter=true), to=hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/63842f73863e9e7b9587486b122e6aae/.tmp/B/053a596b3ce94469af401422641d7f60 2024-11-24T22:14:40,148 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/63842f73863e9e7b9587486b122e6aae/.tmp/C/07bc2a76178f433fb1190a4acc0ae53c is 50, key is test_row_0/C:col10/1732486479160/Put/seqid=0 2024-11-24T22:14:40,154 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45929 is added to blk_1073742030_1206 (size=12301) 2024-11-24T22:14:40,154 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=40087 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=49 2024-11-24T22:14:40,154 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bd6592c37bb2:0-1 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732486460633.63842f73863e9e7b9587486b122e6aae. 2024-11-24T22:14:40,154 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bd6592c37bb2:0-1 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732486460633.63842f73863e9e7b9587486b122e6aae. as already flushing 2024-11-24T22:14:40,155 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bd6592c37bb2:0-1 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732486460633.63842f73863e9e7b9587486b122e6aae. 2024-11-24T22:14:40,155 ERROR [RS_FLUSH_OPERATIONS-regionserver/bd6592c37bb2:0-1 {event_type=RS_FLUSH_REGIONS, pid=49}] handler.RSProcedureHandler(58): pid=49 java.io.IOException: Unable to complete flush {ENCODED => 63842f73863e9e7b9587486b122e6aae, NAME => 'TestAcidGuarantees,,1732486460633.63842f73863e9e7b9587486b122e6aae.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T22:14:40,155 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bd6592c37bb2:0-1 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=49 java.io.IOException: Unable to complete flush {ENCODED => 63842f73863e9e7b9587486b122e6aae, NAME => 'TestAcidGuarantees,,1732486460633.63842f73863e9e7b9587486b122e6aae.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T22:14:40,155 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37701 {}] master.HMaster(4177): Remote procedure failed, pid=49 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 63842f73863e9e7b9587486b122e6aae, NAME => 'TestAcidGuarantees,,1732486460633.63842f73863e9e7b9587486b122e6aae.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 63842f73863e9e7b9587486b122e6aae, NAME => 'TestAcidGuarantees,,1732486460633.63842f73863e9e7b9587486b122e6aae.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T22:14:40,307 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=40087 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=49 2024-11-24T22:14:40,308 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bd6592c37bb2:0-2 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732486460633.63842f73863e9e7b9587486b122e6aae. 2024-11-24T22:14:40,308 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bd6592c37bb2:0-2 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732486460633.63842f73863e9e7b9587486b122e6aae. as already flushing 2024-11-24T22:14:40,308 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bd6592c37bb2:0-2 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732486460633.63842f73863e9e7b9587486b122e6aae. 2024-11-24T22:14:40,308 ERROR [RS_FLUSH_OPERATIONS-regionserver/bd6592c37bb2:0-2 {event_type=RS_FLUSH_REGIONS, pid=49}] handler.RSProcedureHandler(58): pid=49 java.io.IOException: Unable to complete flush {ENCODED => 63842f73863e9e7b9587486b122e6aae, NAME => 'TestAcidGuarantees,,1732486460633.63842f73863e9e7b9587486b122e6aae.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T22:14:40,308 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bd6592c37bb2:0-2 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=49 java.io.IOException: Unable to complete flush {ENCODED => 63842f73863e9e7b9587486b122e6aae, NAME => 'TestAcidGuarantees,,1732486460633.63842f73863e9e7b9587486b122e6aae.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T22:14:40,309 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37701 {}] master.HMaster(4177): Remote procedure failed, pid=49 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 63842f73863e9e7b9587486b122e6aae, NAME => 'TestAcidGuarantees,,1732486460633.63842f73863e9e7b9587486b122e6aae.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 63842f73863e9e7b9587486b122e6aae, NAME => 'TestAcidGuarantees,,1732486460633.63842f73863e9e7b9587486b122e6aae.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T22:14:40,461 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=40087 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=49 2024-11-24T22:14:40,461 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bd6592c37bb2:0-0 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732486460633.63842f73863e9e7b9587486b122e6aae. 2024-11-24T22:14:40,461 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bd6592c37bb2:0-0 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732486460633.63842f73863e9e7b9587486b122e6aae. as already flushing 2024-11-24T22:14:40,462 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bd6592c37bb2:0-0 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732486460633.63842f73863e9e7b9587486b122e6aae. 2024-11-24T22:14:40,462 ERROR [RS_FLUSH_OPERATIONS-regionserver/bd6592c37bb2:0-0 {event_type=RS_FLUSH_REGIONS, pid=49}] handler.RSProcedureHandler(58): pid=49 java.io.IOException: Unable to complete flush {ENCODED => 63842f73863e9e7b9587486b122e6aae, NAME => 'TestAcidGuarantees,,1732486460633.63842f73863e9e7b9587486b122e6aae.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T22:14:40,462 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bd6592c37bb2:0-0 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=49 java.io.IOException: Unable to complete flush {ENCODED => 63842f73863e9e7b9587486b122e6aae, NAME => 'TestAcidGuarantees,,1732486460633.63842f73863e9e7b9587486b122e6aae.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T22:14:40,462 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37701 {}] master.HMaster(4177): Remote procedure failed, pid=49 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 63842f73863e9e7b9587486b122e6aae, NAME => 'TestAcidGuarantees,,1732486460633.63842f73863e9e7b9587486b122e6aae.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 63842f73863e9e7b9587486b122e6aae, NAME => 'TestAcidGuarantees,,1732486460633.63842f73863e9e7b9587486b122e6aae.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T22:14:40,465 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40087 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=63842f73863e9e7b9587486b122e6aae, server=bd6592c37bb2,40087,1732486423713 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-24T22:14:40,465 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40087 {}] ipc.CallRunner(138): callId: 150 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40244 deadline: 1732486540464, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=63842f73863e9e7b9587486b122e6aae, server=bd6592c37bb2,40087,1732486423713 2024-11-24T22:14:40,465 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40087 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=63842f73863e9e7b9587486b122e6aae, server=bd6592c37bb2,40087,1732486423713 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-24T22:14:40,466 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40087 {}] ipc.CallRunner(138): callId: 162 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40206 deadline: 1732486540464, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=63842f73863e9e7b9587486b122e6aae, server=bd6592c37bb2,40087,1732486423713 2024-11-24T22:14:40,466 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732486460633.63842f73863e9e7b9587486b122e6aae., hostname=bd6592c37bb2,40087,1732486423713, seqNum=5 , the old value is region=TestAcidGuarantees,,1732486460633.63842f73863e9e7b9587486b122e6aae., hostname=bd6592c37bb2,40087,1732486423713, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=63842f73863e9e7b9587486b122e6aae, server=bd6592c37bb2,40087,1732486423713 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T22:14:40,466 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732486460633.63842f73863e9e7b9587486b122e6aae., hostname=bd6592c37bb2,40087,1732486423713, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=63842f73863e9e7b9587486b122e6aae, server=bd6592c37bb2,40087,1732486423713 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T22:14:40,466 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732486460633.63842f73863e9e7b9587486b122e6aae., hostname=bd6592c37bb2,40087,1732486423713, seqNum=5 because the exception is null or not the one we care about 2024-11-24T22:14:40,466 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732486460633.63842f73863e9e7b9587486b122e6aae., hostname=bd6592c37bb2,40087,1732486423713, seqNum=5 , the old value is region=TestAcidGuarantees,,1732486460633.63842f73863e9e7b9587486b122e6aae., hostname=bd6592c37bb2,40087,1732486423713, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=63842f73863e9e7b9587486b122e6aae, server=bd6592c37bb2,40087,1732486423713 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T22:14:40,466 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732486460633.63842f73863e9e7b9587486b122e6aae., hostname=bd6592c37bb2,40087,1732486423713, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=63842f73863e9e7b9587486b122e6aae, server=bd6592c37bb2,40087,1732486423713 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T22:14:40,466 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732486460633.63842f73863e9e7b9587486b122e6aae., hostname=bd6592c37bb2,40087,1732486423713, seqNum=5 because the exception is null or not the one we care about 2024-11-24T22:14:40,472 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40087 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=63842f73863e9e7b9587486b122e6aae, server=bd6592c37bb2,40087,1732486423713 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-24T22:14:40,472 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40087 {}] ipc.CallRunner(138): callId: 75 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40202 deadline: 1732486540471, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=63842f73863e9e7b9587486b122e6aae, server=bd6592c37bb2,40087,1732486423713 2024-11-24T22:14:40,473 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732486460633.63842f73863e9e7b9587486b122e6aae., hostname=bd6592c37bb2,40087,1732486423713, seqNum=5 , the old value is region=TestAcidGuarantees,,1732486460633.63842f73863e9e7b9587486b122e6aae., hostname=bd6592c37bb2,40087,1732486423713, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=63842f73863e9e7b9587486b122e6aae, server=bd6592c37bb2,40087,1732486423713 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T22:14:40,473 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732486460633.63842f73863e9e7b9587486b122e6aae., hostname=bd6592c37bb2,40087,1732486423713, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=63842f73863e9e7b9587486b122e6aae, server=bd6592c37bb2,40087,1732486423713 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T22:14:40,473 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732486460633.63842f73863e9e7b9587486b122e6aae., hostname=bd6592c37bb2,40087,1732486423713, seqNum=5 because the exception is null or not the one we care about 2024-11-24T22:14:40,555 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=357 (bloomFilter=true), to=hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/63842f73863e9e7b9587486b122e6aae/.tmp/C/07bc2a76178f433fb1190a4acc0ae53c 2024-11-24T22:14:40,561 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/63842f73863e9e7b9587486b122e6aae/.tmp/A/a09a0cf5571f473d87794d3878c6555e as hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/63842f73863e9e7b9587486b122e6aae/A/a09a0cf5571f473d87794d3878c6555e 2024-11-24T22:14:40,568 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/63842f73863e9e7b9587486b122e6aae/A/a09a0cf5571f473d87794d3878c6555e, entries=200, sequenceid=357, filesize=39.0 K 2024-11-24T22:14:40,574 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/63842f73863e9e7b9587486b122e6aae/.tmp/B/053a596b3ce94469af401422641d7f60 as hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/63842f73863e9e7b9587486b122e6aae/B/053a596b3ce94469af401422641d7f60 2024-11-24T22:14:40,580 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/63842f73863e9e7b9587486b122e6aae/B/053a596b3ce94469af401422641d7f60, entries=150, sequenceid=357, filesize=12.0 K 2024-11-24T22:14:40,582 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/63842f73863e9e7b9587486b122e6aae/.tmp/C/07bc2a76178f433fb1190a4acc0ae53c as hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/63842f73863e9e7b9587486b122e6aae/C/07bc2a76178f433fb1190a4acc0ae53c 2024-11-24T22:14:40,588 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/63842f73863e9e7b9587486b122e6aae/C/07bc2a76178f433fb1190a4acc0ae53c, entries=150, sequenceid=357, filesize=12.0 K 2024-11-24T22:14:40,590 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~134.18 KB/137400, heapSize ~380.25 KB/389376, currentSize=46.96 KB/48090 for 63842f73863e9e7b9587486b122e6aae in 1295ms, sequenceid=357, compaction requested=true 2024-11-24T22:14:40,590 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 63842f73863e9e7b9587486b122e6aae: 2024-11-24T22:14:40,590 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 63842f73863e9e7b9587486b122e6aae:A, priority=-2147483648, current under compaction store size is 1 2024-11-24T22:14:40,590 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-24T22:14:40,590 DEBUG [RS:0;bd6592c37bb2:40087-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-24T22:14:40,590 DEBUG [RS:0;bd6592c37bb2:40087-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-24T22:14:40,590 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 63842f73863e9e7b9587486b122e6aae:B, priority=-2147483648, current under compaction store size is 2 2024-11-24T22:14:40,590 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-24T22:14:40,590 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 63842f73863e9e7b9587486b122e6aae:C, priority=-2147483648, current under compaction store size is 3 2024-11-24T22:14:40,590 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-24T22:14:40,596 DEBUG [RS:0;bd6592c37bb2:40087-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 103141 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-24T22:14:40,596 DEBUG [RS:0;bd6592c37bb2:40087-shortCompactions-0 {}] regionserver.HStore(1541): 63842f73863e9e7b9587486b122e6aae/A is initiating minor compaction (all files) 2024-11-24T22:14:40,596 INFO [RS:0;bd6592c37bb2:40087-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 63842f73863e9e7b9587486b122e6aae/A in TestAcidGuarantees,,1732486460633.63842f73863e9e7b9587486b122e6aae. 2024-11-24T22:14:40,596 INFO [RS:0;bd6592c37bb2:40087-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/63842f73863e9e7b9587486b122e6aae/A/e4f431b6491e4a06b5c36163eb80cecf, hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/63842f73863e9e7b9587486b122e6aae/A/22daac1eb902430cb4be1109cb6cfa80, hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/63842f73863e9e7b9587486b122e6aae/A/a09a0cf5571f473d87794d3878c6555e] into tmpdir=hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/63842f73863e9e7b9587486b122e6aae/.tmp, totalSize=100.7 K 2024-11-24T22:14:40,596 INFO [RS:0;bd6592c37bb2:40087-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=13 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1732486460633.63842f73863e9e7b9587486b122e6aae. 2024-11-24T22:14:40,596 DEBUG [RS:0;bd6592c37bb2:40087-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1732486460633.63842f73863e9e7b9587486b122e6aae. files: [hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/63842f73863e9e7b9587486b122e6aae/A/e4f431b6491e4a06b5c36163eb80cecf, hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/63842f73863e9e7b9587486b122e6aae/A/22daac1eb902430cb4be1109cb6cfa80, hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/63842f73863e9e7b9587486b122e6aae/A/a09a0cf5571f473d87794d3878c6555e] 2024-11-24T22:14:40,597 DEBUG [RS:0;bd6592c37bb2:40087-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37585 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-24T22:14:40,597 DEBUG [RS:0;bd6592c37bb2:40087-longCompactions-0 {}] regionserver.HStore(1541): 63842f73863e9e7b9587486b122e6aae/B is initiating minor compaction (all files) 2024-11-24T22:14:40,597 INFO [RS:0;bd6592c37bb2:40087-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 63842f73863e9e7b9587486b122e6aae/B in TestAcidGuarantees,,1732486460633.63842f73863e9e7b9587486b122e6aae. 2024-11-24T22:14:40,597 DEBUG [RS:0;bd6592c37bb2:40087-shortCompactions-0 {}] compactions.Compactor(225): Compacting e4f431b6491e4a06b5c36163eb80cecf, keycount=150, bloomtype=ROW, size=31.2 K, encoding=NONE, compression=NONE, seqNum=308, earliestPutTs=1732486477695 2024-11-24T22:14:40,597 INFO [RS:0;bd6592c37bb2:40087-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/63842f73863e9e7b9587486b122e6aae/B/c239f7036c684af48076a9457f4f24aa, hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/63842f73863e9e7b9587486b122e6aae/B/e2d876c1c5b849819f4edeeba50d9ad3, hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/63842f73863e9e7b9587486b122e6aae/B/053a596b3ce94469af401422641d7f60] into tmpdir=hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/63842f73863e9e7b9587486b122e6aae/.tmp, totalSize=36.7 K 2024-11-24T22:14:40,597 DEBUG [RS:0;bd6592c37bb2:40087-shortCompactions-0 {}] compactions.Compactor(225): Compacting 22daac1eb902430cb4be1109cb6cfa80, keycount=150, bloomtype=ROW, size=30.5 K, encoding=NONE, compression=NONE, seqNum=319, earliestPutTs=1732486478350 2024-11-24T22:14:40,598 DEBUG [RS:0;bd6592c37bb2:40087-longCompactions-0 {}] compactions.Compactor(225): Compacting c239f7036c684af48076a9457f4f24aa, keycount=150, bloomtype=ROW, size=12.7 K, encoding=NONE, compression=NONE, seqNum=308, earliestPutTs=1732486477695 2024-11-24T22:14:40,598 DEBUG [RS:0;bd6592c37bb2:40087-shortCompactions-0 {}] compactions.Compactor(225): Compacting a09a0cf5571f473d87794d3878c6555e, keycount=200, bloomtype=ROW, size=39.0 K, encoding=NONE, compression=NONE, seqNum=357, earliestPutTs=1732486479160 2024-11-24T22:14:40,599 DEBUG [RS:0;bd6592c37bb2:40087-longCompactions-0 {}] compactions.Compactor(225): Compacting e2d876c1c5b849819f4edeeba50d9ad3, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=319, earliestPutTs=1732486478350 2024-11-24T22:14:40,599 DEBUG [RS:0;bd6592c37bb2:40087-longCompactions-0 {}] compactions.Compactor(225): Compacting 053a596b3ce94469af401422641d7f60, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=357, earliestPutTs=1732486479160 2024-11-24T22:14:40,616 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=40087 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=49 2024-11-24T22:14:40,616 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bd6592c37bb2:0-1 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732486460633.63842f73863e9e7b9587486b122e6aae. 2024-11-24T22:14:40,616 INFO [RS_FLUSH_OPERATIONS-regionserver/bd6592c37bb2:0-1 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.HRegion(2902): Flushing 63842f73863e9e7b9587486b122e6aae 3/3 column families, dataSize=46.96 KB heapSize=140.20 KB 2024-11-24T22:14:40,617 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bd6592c37bb2:0-1 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 63842f73863e9e7b9587486b122e6aae, store=A 2024-11-24T22:14:40,617 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bd6592c37bb2:0-1 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-24T22:14:40,617 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bd6592c37bb2:0-1 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 63842f73863e9e7b9587486b122e6aae, store=B 2024-11-24T22:14:40,617 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bd6592c37bb2:0-1 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-24T22:14:40,617 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bd6592c37bb2:0-1 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 63842f73863e9e7b9587486b122e6aae, store=C 2024-11-24T22:14:40,617 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bd6592c37bb2:0-1 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-24T22:14:40,619 INFO [RS:0;bd6592c37bb2:40087-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=63842f73863e9e7b9587486b122e6aae] 2024-11-24T22:14:40,627 INFO [RS:0;bd6592c37bb2:40087-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 63842f73863e9e7b9587486b122e6aae#B#compaction#169 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-24T22:14:40,627 DEBUG [RS:0;bd6592c37bb2:40087-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/63842f73863e9e7b9587486b122e6aae/.tmp/B/48419645ce0442d981fa746ee68828e1 is 50, key is test_row_0/B:col10/1732486479160/Put/seqid=0 2024-11-24T22:14:40,634 DEBUG [RS:0;bd6592c37bb2:40087-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e2024112402ba1305ef6c4719ac57fea6627fb1ff_63842f73863e9e7b9587486b122e6aae store=[table=TestAcidGuarantees family=A region=63842f73863e9e7b9587486b122e6aae] 2024-11-24T22:14:40,635 DEBUG [RS:0;bd6592c37bb2:40087-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e2024112402ba1305ef6c4719ac57fea6627fb1ff_63842f73863e9e7b9587486b122e6aae, store=[table=TestAcidGuarantees family=A region=63842f73863e9e7b9587486b122e6aae] 2024-11-24T22:14:40,635 DEBUG [RS:0;bd6592c37bb2:40087-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024112402ba1305ef6c4719ac57fea6627fb1ff_63842f73863e9e7b9587486b122e6aae because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=63842f73863e9e7b9587486b122e6aae] 2024-11-24T22:14:40,659 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bd6592c37bb2:0-1 {event_type=RS_FLUSH_REGIONS, pid=49}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024112488cadac551bc49149d727fb10c26b2cc_63842f73863e9e7b9587486b122e6aae is 50, key is test_row_0/A:col10/1732486479310/Put/seqid=0 2024-11-24T22:14:40,676 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45929 is added to blk_1073742031_1207 (size=13085) 2024-11-24T22:14:40,698 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45929 is added to blk_1073742032_1208 (size=4469) 2024-11-24T22:14:40,700 INFO [RS:0;bd6592c37bb2:40087-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 63842f73863e9e7b9587486b122e6aae#A#compaction#168 average throughput is 0.30 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-24T22:14:40,701 DEBUG [RS:0;bd6592c37bb2:40087-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/63842f73863e9e7b9587486b122e6aae/.tmp/A/477957c1db11458f8b66d226b8ee3aeb is 175, key is test_row_0/A:col10/1732486479160/Put/seqid=0 2024-11-24T22:14:40,706 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45929 is added to blk_1073742033_1209 (size=12454) 2024-11-24T22:14:40,711 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bd6592c37bb2:0-1 {event_type=RS_FLUSH_REGIONS, pid=49}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T22:14:40,717 INFO [RS_FLUSH_OPERATIONS-regionserver/bd6592c37bb2:0-1 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024112488cadac551bc49149d727fb10c26b2cc_63842f73863e9e7b9587486b122e6aae to hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024112488cadac551bc49149d727fb10c26b2cc_63842f73863e9e7b9587486b122e6aae 2024-11-24T22:14:40,720 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45929 is added to blk_1073742034_1210 (size=32039) 2024-11-24T22:14:40,721 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bd6592c37bb2:0-1 {event_type=RS_FLUSH_REGIONS, pid=49}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/63842f73863e9e7b9587486b122e6aae/.tmp/A/b45983b335d44ecb8c6b81e63ccb78e4, store: [table=TestAcidGuarantees family=A region=63842f73863e9e7b9587486b122e6aae] 2024-11-24T22:14:40,722 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bd6592c37bb2:0-1 {event_type=RS_FLUSH_REGIONS, pid=49}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/63842f73863e9e7b9587486b122e6aae/.tmp/A/b45983b335d44ecb8c6b81e63ccb78e4 is 175, key is test_row_0/A:col10/1732486479310/Put/seqid=0 2024-11-24T22:14:40,733 DEBUG [RS:0;bd6592c37bb2:40087-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/63842f73863e9e7b9587486b122e6aae/.tmp/A/477957c1db11458f8b66d226b8ee3aeb as hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/63842f73863e9e7b9587486b122e6aae/A/477957c1db11458f8b66d226b8ee3aeb 2024-11-24T22:14:40,739 INFO [RS:0;bd6592c37bb2:40087-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 63842f73863e9e7b9587486b122e6aae/A of 63842f73863e9e7b9587486b122e6aae into 477957c1db11458f8b66d226b8ee3aeb(size=31.3 K), total size for store is 31.3 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-24T22:14:40,739 DEBUG [RS:0;bd6592c37bb2:40087-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 63842f73863e9e7b9587486b122e6aae: 2024-11-24T22:14:40,739 INFO [RS:0;bd6592c37bb2:40087-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732486460633.63842f73863e9e7b9587486b122e6aae., storeName=63842f73863e9e7b9587486b122e6aae/A, priority=13, startTime=1732486480590; duration=0sec 2024-11-24T22:14:40,739 DEBUG [RS:0;bd6592c37bb2:40087-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-24T22:14:40,739 DEBUG [RS:0;bd6592c37bb2:40087-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 63842f73863e9e7b9587486b122e6aae:A 2024-11-24T22:14:40,739 DEBUG [RS:0;bd6592c37bb2:40087-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-24T22:14:40,744 DEBUG [RS:0;bd6592c37bb2:40087-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37585 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-24T22:14:40,744 DEBUG [RS:0;bd6592c37bb2:40087-shortCompactions-0 {}] regionserver.HStore(1541): 63842f73863e9e7b9587486b122e6aae/C is initiating minor compaction (all files) 2024-11-24T22:14:40,744 INFO [RS:0;bd6592c37bb2:40087-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 63842f73863e9e7b9587486b122e6aae/C in TestAcidGuarantees,,1732486460633.63842f73863e9e7b9587486b122e6aae. 2024-11-24T22:14:40,744 INFO [RS:0;bd6592c37bb2:40087-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/63842f73863e9e7b9587486b122e6aae/C/38ae68e5da07426fa5ecf7872ef45fed, hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/63842f73863e9e7b9587486b122e6aae/C/c9bc933b9cc242a4a268f760548c19e2, hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/63842f73863e9e7b9587486b122e6aae/C/07bc2a76178f433fb1190a4acc0ae53c] into tmpdir=hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/63842f73863e9e7b9587486b122e6aae/.tmp, totalSize=36.7 K 2024-11-24T22:14:40,745 DEBUG [RS:0;bd6592c37bb2:40087-shortCompactions-0 {}] compactions.Compactor(225): Compacting 38ae68e5da07426fa5ecf7872ef45fed, keycount=150, bloomtype=ROW, size=12.7 K, encoding=NONE, compression=NONE, seqNum=308, earliestPutTs=1732486477695 2024-11-24T22:14:40,746 DEBUG [RS:0;bd6592c37bb2:40087-shortCompactions-0 {}] compactions.Compactor(225): Compacting c9bc933b9cc242a4a268f760548c19e2, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=319, earliestPutTs=1732486478350 2024-11-24T22:14:40,746 DEBUG [RS:0;bd6592c37bb2:40087-shortCompactions-0 {}] compactions.Compactor(225): Compacting 07bc2a76178f433fb1190a4acc0ae53c, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=357, earliestPutTs=1732486479160 2024-11-24T22:14:40,751 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37701 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=48 2024-11-24T22:14:40,752 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45929 is added to blk_1073742035_1211 (size=31255) 2024-11-24T22:14:40,753 INFO [RS_FLUSH_OPERATIONS-regionserver/bd6592c37bb2:0-1 {event_type=RS_FLUSH_REGIONS, pid=49}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=368, memsize=15.7 K, hasBloomFilter=true, into tmp file hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/63842f73863e9e7b9587486b122e6aae/.tmp/A/b45983b335d44ecb8c6b81e63ccb78e4 2024-11-24T22:14:40,758 INFO [RS:0;bd6592c37bb2:40087-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 63842f73863e9e7b9587486b122e6aae#C#compaction#171 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-24T22:14:40,759 DEBUG [RS:0;bd6592c37bb2:40087-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/63842f73863e9e7b9587486b122e6aae/.tmp/C/e2065151f0e24dcdada181dbbfc3e7f8 is 50, key is test_row_0/C:col10/1732486479160/Put/seqid=0 2024-11-24T22:14:40,767 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bd6592c37bb2:0-1 {event_type=RS_FLUSH_REGIONS, pid=49}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/63842f73863e9e7b9587486b122e6aae/.tmp/B/916c780a9f1c48a7a780b2aa308cc2e3 is 50, key is test_row_0/B:col10/1732486479310/Put/seqid=0 2024-11-24T22:14:40,771 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45929 is added to blk_1073742036_1212 (size=13085) 2024-11-24T22:14:40,778 DEBUG [RS:0;bd6592c37bb2:40087-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/63842f73863e9e7b9587486b122e6aae/.tmp/C/e2065151f0e24dcdada181dbbfc3e7f8 as hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/63842f73863e9e7b9587486b122e6aae/C/e2065151f0e24dcdada181dbbfc3e7f8 2024-11-24T22:14:40,779 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45929 is added to blk_1073742037_1213 (size=12301) 2024-11-24T22:14:40,780 INFO [RS_FLUSH_OPERATIONS-regionserver/bd6592c37bb2:0-1 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=368 (bloomFilter=true), to=hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/63842f73863e9e7b9587486b122e6aae/.tmp/B/916c780a9f1c48a7a780b2aa308cc2e3 2024-11-24T22:14:40,792 INFO [RS:0;bd6592c37bb2:40087-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 63842f73863e9e7b9587486b122e6aae/C of 63842f73863e9e7b9587486b122e6aae into e2065151f0e24dcdada181dbbfc3e7f8(size=12.8 K), total size for store is 12.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-24T22:14:40,792 DEBUG [RS:0;bd6592c37bb2:40087-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 63842f73863e9e7b9587486b122e6aae: 2024-11-24T22:14:40,792 INFO [RS:0;bd6592c37bb2:40087-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732486460633.63842f73863e9e7b9587486b122e6aae., storeName=63842f73863e9e7b9587486b122e6aae/C, priority=13, startTime=1732486480590; duration=0sec 2024-11-24T22:14:40,792 DEBUG [RS:0;bd6592c37bb2:40087-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-24T22:14:40,792 DEBUG [RS:0;bd6592c37bb2:40087-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 63842f73863e9e7b9587486b122e6aae:C 2024-11-24T22:14:40,794 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bd6592c37bb2:0-1 {event_type=RS_FLUSH_REGIONS, pid=49}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/63842f73863e9e7b9587486b122e6aae/.tmp/C/f0e1b796e92c45918e07acbffa3c8650 is 50, key is test_row_0/C:col10/1732486479310/Put/seqid=0 2024-11-24T22:14:40,802 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45929 is added to blk_1073742038_1214 (size=12301) 2024-11-24T22:14:40,804 INFO [RS_FLUSH_OPERATIONS-regionserver/bd6592c37bb2:0-1 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=368 (bloomFilter=true), to=hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/63842f73863e9e7b9587486b122e6aae/.tmp/C/f0e1b796e92c45918e07acbffa3c8650 2024-11-24T22:14:40,808 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bd6592c37bb2:0-1 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/63842f73863e9e7b9587486b122e6aae/.tmp/A/b45983b335d44ecb8c6b81e63ccb78e4 as hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/63842f73863e9e7b9587486b122e6aae/A/b45983b335d44ecb8c6b81e63ccb78e4 2024-11-24T22:14:40,814 INFO [RS_FLUSH_OPERATIONS-regionserver/bd6592c37bb2:0-1 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/63842f73863e9e7b9587486b122e6aae/A/b45983b335d44ecb8c6b81e63ccb78e4, entries=150, sequenceid=368, filesize=30.5 K 2024-11-24T22:14:40,815 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bd6592c37bb2:0-1 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/63842f73863e9e7b9587486b122e6aae/.tmp/B/916c780a9f1c48a7a780b2aa308cc2e3 as hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/63842f73863e9e7b9587486b122e6aae/B/916c780a9f1c48a7a780b2aa308cc2e3 2024-11-24T22:14:40,816 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T22:14:40,816 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T22:14:40,816 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T22:14:40,820 INFO [RS_FLUSH_OPERATIONS-regionserver/bd6592c37bb2:0-1 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/63842f73863e9e7b9587486b122e6aae/B/916c780a9f1c48a7a780b2aa308cc2e3, entries=150, sequenceid=368, filesize=12.0 K 2024-11-24T22:14:40,820 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T22:14:40,821 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bd6592c37bb2:0-1 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/63842f73863e9e7b9587486b122e6aae/.tmp/C/f0e1b796e92c45918e07acbffa3c8650 as hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/63842f73863e9e7b9587486b122e6aae/C/f0e1b796e92c45918e07acbffa3c8650 2024-11-24T22:14:40,821 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T22:14:40,821 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T22:14:40,821 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T22:14:40,821 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T22:14:40,821 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T22:14:40,822 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T22:14:40,822 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T22:14:40,822 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T22:14:40,822 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T22:14:40,823 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T22:14:40,823 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T22:14:40,823 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T22:14:40,823 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T22:14:40,824 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T22:14:40,824 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T22:14:40,824 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T22:14:40,824 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T22:14:40,825 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T22:14:40,825 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T22:14:40,825 INFO [RS_FLUSH_OPERATIONS-regionserver/bd6592c37bb2:0-1 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/63842f73863e9e7b9587486b122e6aae/C/f0e1b796e92c45918e07acbffa3c8650, entries=150, sequenceid=368, filesize=12.0 K 2024-11-24T22:14:40,826 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T22:14:40,826 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T22:14:40,826 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T22:14:40,826 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T22:14:40,826 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T22:14:40,826 INFO [RS_FLUSH_OPERATIONS-regionserver/bd6592c37bb2:0-1 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.HRegion(3140): Finished flush of dataSize ~46.96 KB/48090, heapSize ~140.16 KB/143520, currentSize=0 B/0 for 63842f73863e9e7b9587486b122e6aae in 210ms, sequenceid=368, compaction requested=false 2024-11-24T22:14:40,827 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bd6592c37bb2:0-1 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.HRegion(2603): Flush status journal for 63842f73863e9e7b9587486b122e6aae: 2024-11-24T22:14:40,827 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bd6592c37bb2:0-1 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732486460633.63842f73863e9e7b9587486b122e6aae. 2024-11-24T22:14:40,827 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bd6592c37bb2:0-1 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=49 2024-11-24T22:14:40,827 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T22:14:40,827 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T22:14:40,827 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37701 {}] master.HMaster(4169): Remote procedure done, pid=49 2024-11-24T22:14:40,828 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T22:14:40,828 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T22:14:40,828 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T22:14:40,828 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T22:14:40,829 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T22:14:40,829 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T22:14:40,829 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T22:14:40,829 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T22:14:40,830 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T22:14:40,830 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T22:14:40,830 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T22:14:40,830 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=49, resume processing ppid=48 2024-11-24T22:14:40,831 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=49, ppid=48, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 2.2200 sec 2024-11-24T22:14:40,831 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T22:14:40,831 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T22:14:40,831 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T22:14:40,832 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T22:14:40,832 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T22:14:40,832 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T22:14:40,832 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T22:14:40,833 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T22:14:40,833 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T22:14:40,833 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=48, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=48, table=TestAcidGuarantees in 2.2270 sec 2024-11-24T22:14:40,833 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T22:14:40,833 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T22:14:40,834 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T22:14:40,834 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T22:14:40,834 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T22:14:40,834 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T22:14:40,835 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T22:14:40,835 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T22:14:40,835 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T22:14:40,835 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T22:14:40,835 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T22:14:40,835 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T22:14:40,836 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T22:14:40,836 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T22:14:40,836 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T22:14:40,837 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T22:14:40,837 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T22:14:40,837 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T22:14:40,837 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T22:14:40,837 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T22:14:40,837 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T22:14:40,838 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T22:14:40,838 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T22:14:40,838 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T22:14:40,838 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T22:14:40,839 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T22:14:40,839 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T22:14:40,839 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T22:14:40,839 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T22:14:40,839 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T22:14:40,840 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T22:14:40,840 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T22:14:40,840 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T22:14:40,840 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T22:14:40,840 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T22:14:40,841 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T22:14:40,841 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T22:14:40,841 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T22:14:40,841 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T22:14:40,841 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T22:14:40,842 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T22:14:40,842 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T22:14:40,842 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T22:14:40,842 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T22:14:40,842 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T22:14:40,843 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T22:14:40,843 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T22:14:40,843 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T22:14:40,843 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T22:14:40,843 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T22:14:40,844 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T22:14:40,844 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T22:14:40,844 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T22:14:40,844 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T22:14:40,844 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T22:14:40,845 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T22:14:40,845 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T22:14:40,845 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T22:14:40,845 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T22:14:40,846 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T22:14:40,846 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T22:14:40,846 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T22:14:40,846 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T22:14:40,846 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T22:14:40,847 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T22:14:40,847 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T22:14:40,847 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T22:14:40,847 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T22:14:40,847 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T22:14:40,847 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T22:14:40,848 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T22:14:40,848 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T22:14:40,848 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T22:14:40,848 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T22:14:40,849 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T22:14:40,849 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T22:14:40,849 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T22:14:40,849 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T22:14:40,849 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T22:14:40,849 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T22:14:40,850 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T22:14:40,850 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T22:14:40,850 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T22:14:40,850 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T22:14:40,851 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T22:14:40,851 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T22:14:40,851 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T22:14:40,851 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T22:14:40,852 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T22:14:40,852 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T22:14:40,852 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T22:14:40,853 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T22:14:40,853 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T22:14:40,853 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T22:14:40,853 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T22:14:40,854 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T22:14:40,854 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T22:14:40,854 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T22:14:40,854 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T22:14:40,855 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T22:14:40,857 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T22:14:40,858 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T22:14:40,858 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T22:14:40,858 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T22:14:40,858 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T22:14:40,859 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T22:14:40,859 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T22:14:40,860 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T22:14:40,860 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T22:14:40,860 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T22:14:40,860 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T22:14:40,861 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T22:14:40,861 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T22:14:40,862 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T22:14:40,862 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T22:14:40,862 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T22:14:40,862 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T22:14:40,862 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T22:14:40,862 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T22:14:40,863 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T22:14:40,863 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T22:14:40,863 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T22:14:40,863 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T22:14:40,864 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T22:14:40,864 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T22:14:40,864 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T22:14:40,864 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T22:14:40,864 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T22:14:40,865 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T22:14:40,865 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T22:14:40,865 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T22:14:40,866 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T22:14:40,866 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T22:14:40,866 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T22:14:40,866 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T22:14:40,866 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T22:14:40,867 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T22:14:40,867 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T22:14:40,867 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T22:14:40,867 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T22:14:40,867 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T22:14:40,867 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T22:14:40,868 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T22:14:40,868 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T22:14:40,868 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T22:14:40,868 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T22:14:40,869 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T22:14:40,869 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T22:14:40,869 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T22:14:40,869 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T22:14:40,869 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T22:14:40,869 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T22:14:40,870 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T22:14:40,870 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T22:14:40,870 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T22:14:40,870 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T22:14:40,871 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T22:14:40,871 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T22:14:40,871 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T22:14:40,871 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T22:14:40,871 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T22:14:40,871 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T22:14:40,872 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T22:14:40,872 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T22:14:40,872 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T22:14:40,872 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T22:14:40,872 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T22:14:40,873 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T22:14:40,873 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T22:14:40,873 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T22:14:40,873 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T22:14:40,873 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T22:14:40,874 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T22:14:40,874 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T22:14:40,874 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T22:14:40,875 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T22:14:40,875 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T22:14:40,875 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T22:14:40,876 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T22:14:40,876 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T22:14:40,876 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T22:14:40,877 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T22:14:40,877 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T22:14:40,877 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T22:14:40,877 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T22:14:40,877 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T22:14:40,878 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T22:14:40,878 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T22:14:40,878 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T22:14:40,878 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T22:14:40,879 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T22:14:40,879 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T22:14:40,879 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T22:14:40,880 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T22:14:40,880 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T22:14:40,881 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T22:14:40,881 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T22:14:40,881 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T22:14:40,881 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T22:14:40,881 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T22:14:40,882 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T22:14:40,882 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T22:14:40,882 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T22:14:40,882 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T22:14:40,882 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T22:14:40,883 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T22:14:40,883 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T22:14:40,883 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T22:14:40,883 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T22:14:40,883 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T22:14:40,884 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T22:14:40,884 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T22:14:40,884 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T22:14:40,884 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T22:14:40,884 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T22:14:40,884 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T22:14:40,885 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T22:14:40,885 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T22:14:40,885 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T22:14:40,885 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T22:14:40,885 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T22:14:40,886 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T22:14:40,886 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T22:14:40,886 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T22:14:40,886 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T22:14:40,886 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T22:14:40,887 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T22:14:40,887 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T22:14:40,887 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T22:14:40,887 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T22:14:40,888 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T22:14:40,888 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T22:14:40,888 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T22:14:40,888 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T22:14:40,888 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T22:14:40,889 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T22:14:40,889 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T22:14:40,889 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T22:14:40,889 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T22:14:40,889 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T22:14:40,890 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T22:14:40,890 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T22:14:40,890 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T22:14:40,890 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T22:14:40,891 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T22:14:40,891 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T22:14:40,891 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T22:14:40,891 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T22:14:40,892 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T22:14:40,892 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T22:14:40,892 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T22:14:40,893 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T22:14:40,893 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T22:14:40,893 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T22:14:40,893 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T22:14:40,894 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T22:14:40,894 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T22:14:40,894 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T22:14:40,894 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T22:14:40,895 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T22:14:40,895 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T22:14:40,895 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T22:14:40,895 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T22:14:40,896 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T22:14:40,896 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T22:14:40,896 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T22:14:40,896 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T22:14:40,896 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T22:14:40,897 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T22:14:40,897 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T22:14:40,897 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T22:14:40,897 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T22:14:40,897 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T22:14:40,897 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T22:14:40,898 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T22:14:40,898 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T22:14:40,898 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T22:14:40,898 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T22:14:40,898 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T22:14:40,899 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T22:14:40,899 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T22:14:40,899 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T22:14:40,899 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T22:14:40,899 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T22:14:40,900 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T22:14:40,900 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T22:14:40,900 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T22:14:40,900 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T22:14:40,900 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T22:14:40,901 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T22:14:40,901 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T22:14:40,901 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T22:14:40,901 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T22:14:40,901 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T22:14:40,902 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T22:14:40,902 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T22:14:40,902 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T22:14:40,902 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T22:14:40,902 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T22:14:40,903 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T22:14:40,903 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T22:14:40,903 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T22:14:40,903 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T22:14:40,903 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T22:14:40,904 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T22:14:40,904 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T22:14:40,904 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T22:14:40,904 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T22:14:40,904 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T22:14:40,904 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T22:14:40,905 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T22:14:40,905 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T22:14:40,905 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T22:14:40,905 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T22:14:40,905 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T22:14:40,906 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T22:14:40,906 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T22:14:40,906 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T22:14:40,906 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T22:14:40,906 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T22:14:40,906 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T22:14:40,907 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T22:14:40,907 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T22:14:40,907 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T22:14:40,907 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T22:14:40,908 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T22:14:40,908 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T22:14:40,908 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T22:14:40,908 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T22:14:40,909 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T22:14:40,909 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T22:14:40,909 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T22:14:40,909 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T22:14:40,910 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T22:14:40,910 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T22:14:40,910 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T22:14:40,910 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T22:14:40,910 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T22:14:40,910 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T22:14:40,911 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T22:14:40,911 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T22:14:40,911 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T22:14:40,911 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T22:14:40,912 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T22:14:40,912 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T22:14:40,912 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T22:14:40,912 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T22:14:40,912 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T22:14:40,912 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T22:14:40,913 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T22:14:40,913 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T22:14:40,913 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T22:14:40,913 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T22:14:40,913 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T22:14:40,914 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T22:14:40,914 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T22:14:40,914 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T22:14:40,914 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T22:14:40,914 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T22:14:40,915 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T22:14:40,915 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T22:14:40,915 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T22:14:40,915 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T22:14:40,915 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T22:14:40,916 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T22:14:40,916 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T22:14:40,916 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T22:14:40,916 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T22:14:40,916 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T22:14:40,917 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T22:14:40,917 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T22:14:40,917 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T22:14:40,917 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T22:14:40,917 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T22:14:40,918 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T22:14:40,918 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T22:14:40,918 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T22:14:40,918 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T22:14:40,918 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T22:14:40,919 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T22:14:40,919 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T22:14:40,919 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T22:14:40,919 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T22:14:40,919 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T22:14:40,920 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T22:14:40,920 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T22:14:40,920 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T22:14:40,920 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T22:14:40,920 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T22:14:40,921 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T22:14:40,921 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T22:14:40,921 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T22:14:40,922 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T22:14:40,922 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T22:14:40,922 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T22:14:40,923 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T22:14:40,923 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T22:14:40,924 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T22:14:40,924 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T22:14:40,924 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T22:14:40,924 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T22:14:40,925 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T22:14:40,925 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T22:14:40,925 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T22:14:40,925 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T22:14:40,926 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T22:14:40,926 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T22:14:40,926 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T22:14:40,926 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T22:14:40,927 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T22:14:40,927 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T22:14:40,927 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T22:14:40,927 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T22:14:40,927 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T22:14:40,928 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T22:14:40,928 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T22:14:40,928 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T22:14:40,929 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T22:14:40,929 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T22:14:40,929 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T22:14:40,929 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T22:14:40,929 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T22:14:40,929 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T22:14:40,930 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T22:14:40,930 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T22:14:40,930 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T22:14:40,930 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T22:14:40,931 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T22:14:40,931 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T22:14:40,931 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T22:14:40,931 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T22:14:40,931 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T22:14:40,932 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T22:14:40,932 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T22:14:40,932 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T22:14:40,932 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T22:14:40,932 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T22:14:40,933 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T22:14:40,933 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T22:14:40,933 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T22:14:40,933 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T22:14:40,933 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T22:14:40,933 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T22:14:40,934 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T22:14:40,934 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T22:14:40,934 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T22:14:40,934 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T22:14:40,935 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T22:14:40,935 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T22:14:40,935 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T22:14:40,935 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T22:14:40,935 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T22:14:40,935 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T22:14:40,936 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T22:14:40,936 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T22:14:40,936 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T22:14:40,936 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T22:14:40,937 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T22:14:40,937 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T22:14:40,937 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T22:14:40,937 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T22:14:40,937 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T22:14:40,937 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T22:14:40,938 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T22:14:40,938 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T22:14:40,938 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T22:14:40,938 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T22:14:40,938 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T22:14:40,939 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T22:14:40,939 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T22:14:40,939 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T22:14:40,939 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T22:14:40,939 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T22:14:40,940 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T22:14:40,940 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T22:14:40,940 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T22:14:40,940 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T22:14:40,940 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T22:14:40,941 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T22:14:40,941 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T22:14:40,941 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T22:14:40,941 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T22:14:40,942 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T22:14:40,942 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T22:14:40,942 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T22:14:40,942 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T22:14:40,942 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T22:14:40,943 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T22:14:40,943 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T22:14:40,943 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T22:14:40,943 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T22:14:40,944 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T22:14:40,944 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T22:14:40,944 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T22:14:40,944 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T22:14:40,944 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T22:14:40,945 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T22:14:40,945 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T22:14:40,945 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T22:14:40,945 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T22:14:40,945 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T22:14:40,946 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T22:14:40,946 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T22:14:40,946 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T22:14:40,946 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T22:14:40,946 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T22:14:40,947 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T22:14:40,947 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T22:14:40,947 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T22:14:40,947 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T22:14:40,947 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T22:14:40,948 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T22:14:40,948 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T22:14:40,948 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T22:14:40,948 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T22:14:40,949 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T22:14:40,949 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T22:14:40,949 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T22:14:40,949 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T22:14:40,949 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T22:14:40,950 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T22:14:40,950 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T22:14:40,950 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T22:14:40,950 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T22:14:40,950 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T22:14:40,951 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T22:14:40,951 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T22:14:40,951 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T22:14:40,951 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T22:14:40,951 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T22:14:40,952 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T22:14:40,952 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T22:14:40,952 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T22:14:40,953 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T22:14:40,953 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T22:14:40,953 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T22:14:40,953 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T22:14:40,954 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T22:14:40,955 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T22:14:40,955 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T22:14:40,955 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T22:14:40,971 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T22:14:40,972 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T22:14:40,972 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T22:14:40,972 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T22:14:40,972 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T22:14:40,973 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T22:14:40,973 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T22:14:40,973 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T22:14:40,973 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T22:14:40,974 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T22:14:40,974 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T22:14:40,974 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T22:14:40,974 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T22:14:40,975 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T22:14:40,975 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T22:14:40,975 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T22:14:40,975 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T22:14:40,976 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T22:14:40,976 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T22:14:40,976 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T22:14:40,977 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T22:14:40,977 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T22:14:40,977 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T22:14:40,978 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T22:14:40,979 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T22:14:40,979 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T22:14:40,979 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T22:14:40,980 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T22:14:40,980 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T22:14:40,980 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T22:14:40,981 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T22:14:40,981 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T22:14:40,982 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T22:14:40,982 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T22:14:40,982 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T22:14:40,982 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T22:14:40,982 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T22:14:40,983 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T22:14:40,983 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T22:14:40,983 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T22:14:40,984 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T22:14:40,984 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T22:14:40,984 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T22:14:40,984 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T22:14:40,985 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T22:14:40,985 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T22:14:40,985 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T22:14:40,985 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T22:14:40,985 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T22:14:40,986 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T22:14:40,986 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T22:14:40,986 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T22:14:40,986 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T22:14:40,987 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T22:14:40,987 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T22:14:40,987 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T22:14:40,987 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T22:14:40,988 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T22:14:40,988 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T22:14:40,988 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T22:14:40,988 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T22:14:40,989 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T22:14:40,989 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T22:14:40,989 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T22:14:40,989 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T22:14:40,990 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T22:14:40,990 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T22:14:40,990 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T22:14:40,991 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T22:14:40,991 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T22:14:40,991 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T22:14:40,991 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T22:14:40,992 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T22:14:40,992 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T22:14:40,992 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T22:14:40,992 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T22:14:40,993 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T22:14:40,993 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T22:14:40,993 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T22:14:40,993 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T22:14:40,993 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T22:14:40,994 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T22:14:40,994 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T22:14:40,994 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T22:14:40,995 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T22:14:40,995 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T22:14:40,995 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T22:14:40,995 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T22:14:40,995 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T22:14:40,995 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T22:14:40,996 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T22:14:40,996 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T22:14:40,996 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T22:14:40,997 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T22:14:40,997 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T22:14:40,997 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T22:14:40,997 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T22:14:40,998 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T22:14:40,998 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T22:14:40,998 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T22:14:40,998 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T22:14:40,998 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T22:14:40,999 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T22:14:40,999 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T22:14:40,999 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T22:14:40,999 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T22:14:41,000 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T22:14:41,000 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T22:14:41,000 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T22:14:41,001 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T22:14:41,001 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T22:14:41,001 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T22:14:41,001 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T22:14:41,002 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T22:14:41,002 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T22:14:41,002 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T22:14:41,002 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T22:14:41,002 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T22:14:41,003 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T22:14:41,003 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T22:14:41,003 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T22:14:41,004 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T22:14:41,004 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T22:14:41,004 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T22:14:41,004 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T22:14:41,005 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T22:14:41,005 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T22:14:41,005 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T22:14:41,005 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T22:14:41,006 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T22:14:41,006 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T22:14:41,006 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T22:14:41,006 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T22:14:41,007 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T22:14:41,007 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T22:14:41,007 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T22:14:41,007 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T22:14:41,007 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T22:14:41,008 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T22:14:41,008 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T22:14:41,008 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T22:14:41,009 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T22:14:41,009 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T22:14:41,009 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T22:14:41,009 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T22:14:41,010 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T22:14:41,010 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T22:14:41,010 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T22:14:41,011 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T22:14:41,011 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T22:14:41,011 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T22:14:41,011 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T22:14:41,012 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T22:14:41,012 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T22:14:41,012 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T22:14:41,012 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T22:14:41,013 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T22:14:41,013 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T22:14:41,013 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T22:14:41,013 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T22:14:41,013 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T22:14:41,014 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T22:14:41,014 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T22:14:41,014 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T22:14:41,015 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T22:14:41,015 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T22:14:41,015 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T22:14:41,016 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T22:14:41,016 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T22:14:41,016 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T22:14:41,017 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T22:14:41,017 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T22:14:41,017 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T22:14:41,018 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T22:14:41,018 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T22:14:41,018 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T22:14:41,018 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T22:14:41,018 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T22:14:41,019 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T22:14:41,019 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T22:14:41,019 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T22:14:41,019 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T22:14:41,020 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T22:14:41,020 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T22:14:41,020 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T22:14:41,021 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T22:14:41,021 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T22:14:41,021 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T22:14:41,021 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T22:14:41,021 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T22:14:41,022 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T22:14:41,022 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T22:14:41,022 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T22:14:41,022 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T22:14:41,023 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T22:14:41,023 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T22:14:41,024 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T22:14:41,024 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T22:14:41,025 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T22:14:41,025 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T22:14:41,025 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T22:14:41,026 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T22:14:41,026 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T22:14:41,026 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T22:14:41,027 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T22:14:41,027 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T22:14:41,028 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T22:14:41,028 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T22:14:41,028 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T22:14:41,028 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T22:14:41,029 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T22:14:41,029 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T22:14:41,029 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T22:14:41,030 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T22:14:41,030 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T22:14:41,031 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T22:14:41,031 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T22:14:41,031 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T22:14:41,031 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T22:14:41,032 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T22:14:41,032 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T22:14:41,032 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T22:14:41,032 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T22:14:41,032 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T22:14:41,033 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T22:14:41,033 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T22:14:41,033 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T22:14:41,033 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T22:14:41,034 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T22:14:41,034 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T22:14:41,034 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T22:14:41,035 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T22:14:41,035 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T22:14:41,035 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T22:14:41,035 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T22:14:41,035 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T22:14:41,036 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T22:14:41,036 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T22:14:41,036 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T22:14:41,036 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T22:14:41,037 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T22:14:41,037 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T22:14:41,037 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T22:14:41,037 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T22:14:41,037 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T22:14:41,038 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T22:14:41,038 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T22:14:41,038 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T22:14:41,039 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T22:14:41,039 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T22:14:41,039 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T22:14:41,039 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T22:14:41,039 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T22:14:41,039 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T22:14:41,040 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T22:14:41,040 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T22:14:41,041 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T22:14:41,041 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T22:14:41,041 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T22:14:41,041 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T22:14:41,042 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T22:14:41,042 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T22:14:41,042 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T22:14:41,042 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T22:14:41,043 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T22:14:41,043 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T22:14:41,043 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T22:14:41,044 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T22:14:41,044 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T22:14:41,044 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T22:14:41,044 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T22:14:41,045 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T22:14:41,045 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T22:14:41,045 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T22:14:41,046 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T22:14:41,046 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T22:14:41,047 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T22:14:41,047 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T22:14:41,047 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T22:14:41,047 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T22:14:41,048 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T22:14:41,048 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T22:14:41,048 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T22:14:41,049 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T22:14:41,049 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T22:14:41,049 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T22:14:41,049 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T22:14:41,050 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T22:14:41,050 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T22:14:41,050 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T22:14:41,050 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T22:14:41,051 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T22:14:41,051 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T22:14:41,052 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T22:14:41,052 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T22:14:41,052 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T22:14:41,053 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T22:14:41,053 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T22:14:41,054 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T22:14:41,054 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T22:14:41,055 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T22:14:41,055 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T22:14:41,055 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T22:14:41,055 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T22:14:41,056 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T22:14:41,057 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T22:14:41,057 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T22:14:41,058 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T22:14:41,058 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T22:14:41,058 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T22:14:41,058 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T22:14:41,059 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T22:14:41,059 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T22:14:41,060 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T22:14:41,060 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T22:14:41,060 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T22:14:41,061 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T22:14:41,061 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T22:14:41,061 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T22:14:41,061 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T22:14:41,061 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T22:14:41,062 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T22:14:41,062 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T22:14:41,063 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T22:14:41,063 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T22:14:41,063 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T22:14:41,063 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T22:14:41,064 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T22:14:41,064 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T22:14:41,064 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T22:14:41,064 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T22:14:41,065 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T22:14:41,065 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T22:14:41,065 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T22:14:41,065 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T22:14:41,066 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T22:14:41,066 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T22:14:41,066 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T22:14:41,066 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T22:14:41,067 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T22:14:41,067 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T22:14:41,067 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T22:14:41,068 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T22:14:41,068 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T22:14:41,068 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T22:14:41,069 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T22:14:41,069 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T22:14:41,069 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T22:14:41,069 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T22:14:41,069 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T22:14:41,070 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T22:14:41,070 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T22:14:41,070 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T22:14:41,071 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T22:14:41,070 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T22:14:41,071 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T22:14:41,071 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T22:14:41,072 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T22:14:41,072 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T22:14:41,072 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T22:14:41,072 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T22:14:41,073 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T22:14:41,073 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T22:14:41,073 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T22:14:41,073 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T22:14:41,074 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T22:14:41,074 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T22:14:41,074 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T22:14:41,075 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T22:14:41,075 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T22:14:41,076 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T22:14:41,076 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T22:14:41,076 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T22:14:41,076 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T22:14:41,077 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T22:14:41,077 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T22:14:41,078 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T22:14:41,078 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T22:14:41,079 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T22:14:41,079 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T22:14:41,080 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T22:14:41,080 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T22:14:41,080 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T22:14:41,081 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T22:14:41,081 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T22:14:41,082 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T22:14:41,083 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T22:14:41,083 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T22:14:41,083 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T22:14:41,084 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T22:14:41,084 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T22:14:41,084 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T22:14:41,085 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T22:14:41,085 DEBUG [RS:0;bd6592c37bb2:40087-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/63842f73863e9e7b9587486b122e6aae/.tmp/B/48419645ce0442d981fa746ee68828e1 as hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/63842f73863e9e7b9587486b122e6aae/B/48419645ce0442d981fa746ee68828e1 2024-11-24T22:14:41,085 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T22:14:41,086 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T22:14:41,086 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T22:14:41,086 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T22:14:41,087 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T22:14:41,087 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T22:14:41,087 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T22:14:41,088 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T22:14:41,088 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T22:14:41,088 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T22:14:41,089 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T22:14:41,090 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T22:14:41,090 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T22:14:41,091 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T22:14:41,092 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T22:14:41,092 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T22:14:41,093 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T22:14:41,094 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T22:14:41,094 INFO [RS:0;bd6592c37bb2:40087-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 63842f73863e9e7b9587486b122e6aae/B of 63842f73863e9e7b9587486b122e6aae into 48419645ce0442d981fa746ee68828e1(size=12.8 K), total size for store is 24.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-24T22:14:41,095 DEBUG [RS:0;bd6592c37bb2:40087-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 63842f73863e9e7b9587486b122e6aae: 2024-11-24T22:14:41,095 INFO [RS:0;bd6592c37bb2:40087-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732486460633.63842f73863e9e7b9587486b122e6aae., storeName=63842f73863e9e7b9587486b122e6aae/B, priority=13, startTime=1732486480590; duration=0sec 2024-11-24T22:14:41,095 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T22:14:41,095 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T22:14:41,095 DEBUG [RS:0;bd6592c37bb2:40087-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-24T22:14:41,095 DEBUG [RS:0;bd6592c37bb2:40087-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 63842f73863e9e7b9587486b122e6aae:B 2024-11-24T22:14:41,095 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T22:14:41,096 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T22:14:41,096 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T22:14:41,097 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T22:14:41,097 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T22:14:41,097 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T22:14:41,098 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T22:14:41,098 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T22:14:41,098 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T22:14:41,099 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T22:14:41,099 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T22:14:41,099 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T22:14:41,100 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T22:14:41,100 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T22:14:41,101 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T22:14:41,101 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T22:14:41,102 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T22:14:41,102 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T22:14:41,102 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T22:14:41,102 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T22:14:41,103 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T22:14:41,103 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T22:14:41,104 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T22:14:41,104 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T22:14:41,104 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T22:14:41,105 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T22:14:41,105 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T22:14:41,106 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T22:14:41,106 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T22:14:41,107 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T22:14:41,107 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T22:14:41,107 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T22:14:41,107 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T22:14:41,107 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T22:14:41,108 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T22:14:41,109 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T22:14:41,109 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T22:14:41,110 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T22:14:41,111 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T22:14:41,112 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T22:14:41,112 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T22:14:41,113 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T22:14:41,113 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T22:14:41,114 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T22:14:41,114 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T22:14:41,115 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T22:14:41,115 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T22:14:41,115 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T22:14:41,115 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T22:14:41,116 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T22:14:41,116 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T22:14:41,117 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T22:14:41,118 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T22:14:41,118 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T22:14:41,119 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T22:14:41,119 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T22:14:41,119 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T22:14:41,119 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T22:14:41,120 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T22:14:41,121 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T22:14:41,121 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T22:14:41,121 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T22:14:41,121 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T22:14:41,122 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T22:14:41,122 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T22:14:41,123 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T22:14:41,123 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T22:14:41,123 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T22:14:41,123 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T22:14:41,123 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T22:14:41,124 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T22:14:41,124 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T22:14:41,125 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T22:14:41,125 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T22:14:41,125 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T22:14:41,125 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T22:14:41,126 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T22:14:41,126 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T22:14:41,126 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T22:14:41,126 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T22:14:41,127 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T22:14:41,127 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T22:14:41,127 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T22:14:41,127 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T22:14:41,128 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T22:14:41,128 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T22:14:41,128 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T22:14:41,128 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T22:14:41,129 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T22:14:41,129 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T22:14:41,129 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T22:14:41,130 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T22:14:41,130 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T22:14:41,130 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T22:14:41,130 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T22:14:41,131 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T22:14:41,131 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T22:14:41,131 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T22:14:41,132 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T22:14:41,132 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T22:14:41,132 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T22:14:41,132 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T22:14:41,133 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T22:14:41,133 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T22:14:41,133 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T22:14:41,133 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T22:14:41,133 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T22:14:41,134 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T22:14:41,134 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T22:14:41,134 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T22:14:41,135 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T22:14:41,135 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T22:14:41,135 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T22:14:41,135 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T22:14:41,136 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T22:14:41,136 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T22:14:41,136 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T22:14:41,136 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T22:14:41,137 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T22:14:41,137 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T22:14:41,137 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T22:14:41,138 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T22:14:41,138 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T22:14:41,138 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T22:14:41,138 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T22:14:41,138 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T22:14:41,139 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T22:14:41,139 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T22:14:41,139 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T22:14:41,140 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T22:14:41,140 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T22:14:41,141 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T22:14:41,141 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T22:14:41,142 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T22:14:41,142 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T22:14:41,142 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T22:14:41,142 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T22:14:41,143 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T22:14:41,143 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T22:14:41,144 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T22:14:41,144 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T22:14:41,144 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T22:14:41,144 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T22:14:41,145 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T22:14:41,145 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T22:14:41,145 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T22:14:41,145 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T22:14:41,146 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T22:14:41,146 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T22:14:41,146 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T22:14:41,146 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T22:14:41,146 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T22:14:41,147 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T22:14:41,147 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T22:14:41,148 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T22:14:41,148 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T22:14:41,148 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T22:14:41,148 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T22:14:41,148 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T22:14:41,149 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T22:14:41,149 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T22:14:41,149 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T22:14:41,150 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T22:14:41,150 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T22:14:41,150 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T22:14:41,150 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T22:14:41,150 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T22:14:41,151 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T22:14:41,151 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T22:14:41,151 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T22:14:41,151 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T22:14:41,152 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T22:14:41,152 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T22:14:41,152 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T22:14:41,152 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T22:14:41,153 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T22:14:41,153 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T22:14:41,153 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T22:14:41,153 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T22:14:41,154 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T22:14:41,154 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T22:14:41,154 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T22:14:41,155 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T22:14:41,155 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T22:14:41,155 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T22:14:41,155 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T22:14:41,156 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T22:14:41,156 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T22:14:41,156 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T22:14:41,156 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T22:14:41,156 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T22:14:41,157 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T22:14:41,157 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T22:14:41,157 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T22:14:41,158 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T22:14:41,159 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T22:14:41,160 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T22:14:41,160 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T22:14:41,161 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T22:14:41,161 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T22:14:41,161 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T22:14:41,162 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T22:14:41,162 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T22:14:41,163 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T22:14:41,163 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T22:14:41,163 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T22:14:41,163 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T22:14:41,164 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T22:14:41,164 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T22:14:41,164 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T22:14:41,164 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T22:14:41,165 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T22:14:41,165 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T22:14:41,165 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T22:14:41,165 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T22:14:41,165 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T22:14:41,166 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T22:14:41,166 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T22:14:41,166 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T22:14:41,167 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T22:14:41,167 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T22:14:41,167 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T22:14:41,168 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T22:14:41,168 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T22:14:41,168 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T22:14:41,168 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T22:14:41,168 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T22:14:41,169 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T22:14:41,169 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T22:14:41,169 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T22:14:41,169 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T22:14:41,170 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T22:14:41,170 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T22:14:41,170 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T22:14:41,170 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T22:14:41,170 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T22:14:41,171 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T22:14:41,171 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T22:14:41,171 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T22:14:41,171 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T22:14:41,171 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T22:14:41,172 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T22:14:41,172 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T22:14:41,172 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T22:14:41,173 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T22:14:41,173 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T22:14:41,173 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T22:14:41,173 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T22:14:41,174 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T22:14:41,174 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T22:14:41,174 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T22:14:41,174 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T22:14:41,175 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T22:14:41,175 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T22:14:41,175 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T22:14:41,176 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T22:14:41,176 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T22:14:41,176 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T22:14:41,176 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T22:14:41,177 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T22:14:41,177 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T22:14:41,177 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T22:14:41,177 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T22:14:41,178 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T22:14:41,178 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T22:14:41,178 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T22:14:41,178 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T22:14:41,178 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T22:14:41,179 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T22:14:41,179 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T22:14:41,179 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T22:14:41,180 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T22:14:41,180 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T22:14:41,180 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T22:14:41,180 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T22:14:41,180 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T22:14:41,181 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T22:14:41,181 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T22:14:41,181 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T22:14:41,181 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T22:14:41,182 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T22:14:41,183 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T22:14:41,183 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T22:14:41,184 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T22:14:41,184 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T22:14:41,185 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T22:14:41,185 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T22:14:41,185 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T22:14:41,185 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T22:14:41,186 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T22:14:41,186 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T22:14:41,186 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T22:14:41,186 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T22:14:41,187 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T22:14:41,187 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T22:14:41,187 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T22:14:41,187 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T22:14:41,188 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T22:14:41,188 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T22:14:41,188 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T22:14:41,188 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T22:14:41,189 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T22:14:41,189 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T22:14:41,189 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T22:14:41,189 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T22:14:41,190 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T22:14:41,190 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T22:14:41,190 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T22:14:41,191 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T22:14:41,191 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T22:14:41,191 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T22:14:41,191 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T22:14:41,192 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T22:14:41,192 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T22:14:41,192 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T22:14:41,192 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T22:14:41,193 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T22:14:41,195 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T22:14:41,197 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T22:14:41,197 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T22:14:41,197 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T22:14:41,198 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T22:14:41,198 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T22:14:41,199 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T22:14:41,200 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T22:14:41,200 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T22:14:41,201 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T22:14:41,201 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T22:14:41,201 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T22:14:41,202 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T22:14:41,202 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T22:14:41,202 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T22:14:41,203 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T22:14:41,203 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T22:14:41,203 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T22:14:41,204 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T22:14:41,204 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T22:14:41,204 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T22:14:41,204 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T22:14:41,205 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T22:14:41,205 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T22:14:41,205 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T22:14:41,206 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T22:14:41,206 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T22:14:41,206 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T22:14:41,207 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T22:14:41,207 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T22:14:41,207 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T22:14:41,208 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T22:14:41,208 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T22:14:41,208 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T22:14:41,208 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T22:14:41,209 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T22:14:41,209 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T22:14:41,209 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T22:14:41,210 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T22:14:41,210 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T22:14:41,211 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T22:14:41,211 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T22:14:41,211 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T22:14:41,212 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T22:14:41,212 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T22:14:41,212 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T22:14:41,213 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T22:14:41,214 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T22:14:41,215 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T22:14:41,215 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T22:14:41,215 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T22:14:41,215 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T22:14:41,216 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T22:14:41,216 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T22:14:41,216 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T22:14:41,217 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T22:14:41,217 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T22:14:41,217 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T22:14:41,217 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T22:14:41,218 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T22:14:41,218 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T22:14:41,218 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T22:14:41,219 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T22:14:41,219 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T22:14:41,219 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T22:14:41,219 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T22:14:41,220 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T22:14:41,220 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T22:14:41,220 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T22:14:41,221 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T22:14:41,221 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T22:14:41,221 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T22:14:41,222 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T22:14:41,222 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T22:14:41,223 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T22:14:41,223 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T22:14:41,223 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T22:14:41,223 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T22:14:41,223 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T22:14:41,224 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T22:14:41,224 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T22:14:41,224 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T22:14:41,224 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T22:14:41,225 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T22:14:41,225 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T22:14:41,225 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T22:14:41,226 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T22:14:41,226 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T22:14:41,226 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T22:14:41,226 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T22:14:41,226 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T22:14:41,227 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T22:14:41,227 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T22:14:41,227 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T22:14:41,228 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T22:14:41,228 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T22:14:41,228 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T22:14:41,228 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T22:14:41,229 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T22:14:41,229 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T22:14:41,229 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T22:14:41,229 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T22:14:41,230 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T22:14:41,230 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T22:14:41,230 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T22:14:41,230 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T22:14:41,230 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T22:14:41,231 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T22:14:41,231 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T22:14:41,231 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T22:14:41,232 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T22:14:41,232 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T22:14:41,232 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T22:14:41,232 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T22:14:41,232 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T22:14:41,233 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T22:14:41,233 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T22:14:41,233 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T22:14:41,233 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T22:14:41,234 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T22:14:41,234 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T22:14:41,234 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T22:14:41,234 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T22:14:41,235 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T22:14:41,235 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T22:14:41,235 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T22:14:41,235 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T22:14:41,236 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T22:14:41,236 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T22:14:41,236 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T22:14:41,236 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T22:14:41,236 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T22:14:41,236 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T22:14:41,237 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T22:14:41,237 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T22:14:41,237 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T22:14:41,238 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T22:14:41,238 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T22:14:41,238 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T22:14:41,238 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T22:14:41,238 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T22:14:41,239 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T22:14:41,239 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T22:14:41,240 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T22:14:41,240 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T22:14:41,240 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T22:14:41,240 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T22:14:41,241 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T22:14:41,241 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T22:14:41,241 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T22:14:41,242 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T22:14:41,242 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T22:14:41,242 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T22:14:41,242 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T22:14:41,243 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T22:14:41,243 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T22:14:41,243 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T22:14:41,243 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T22:14:41,244 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T22:14:41,244 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T22:14:41,244 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T22:14:41,244 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T22:14:41,245 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T22:14:41,245 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T22:14:41,245 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T22:14:41,245 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T22:14:41,246 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T22:14:41,246 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T22:14:41,246 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T22:14:41,246 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T22:14:41,246 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T22:14:41,247 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T22:14:41,247 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T22:14:41,247 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T22:14:41,247 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T22:14:41,248 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T22:14:41,248 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T22:14:41,248 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T22:14:41,248 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T22:14:41,249 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T22:14:41,249 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T22:14:41,249 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T22:14:41,250 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T22:14:41,250 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T22:14:41,250 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T22:14:41,251 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T22:14:41,252 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T22:14:41,252 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T22:14:41,252 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T22:14:41,253 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T22:14:41,253 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T22:14:41,253 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T22:14:41,254 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T22:14:41,254 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T22:14:41,255 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T22:14:41,255 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T22:14:41,255 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T22:14:41,256 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T22:14:41,256 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T22:14:41,257 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T22:14:41,257 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T22:14:41,257 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T22:14:41,257 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T22:14:41,257 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T22:14:41,258 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T22:14:41,258 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T22:14:41,258 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T22:14:41,258 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T22:14:41,258 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T22:14:41,259 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T22:14:41,259 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T22:14:41,260 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T22:14:41,260 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T22:14:41,260 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T22:14:41,261 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T22:14:41,261 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T22:14:41,261 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T22:14:41,262 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T22:14:41,262 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T22:14:41,262 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T22:14:41,262 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T22:14:41,263 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T22:14:41,263 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T22:14:41,263 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T22:14:41,263 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T22:14:41,264 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T22:14:41,264 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T22:14:41,264 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T22:14:41,264 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T22:14:41,265 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T22:14:41,265 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T22:14:41,265 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T22:14:41,265 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T22:14:41,266 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T22:14:41,266 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T22:14:41,266 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T22:14:41,266 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T22:14:41,267 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T22:14:41,267 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T22:14:41,267 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T22:14:41,267 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T22:14:41,267 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T22:14:41,268 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T22:14:41,268 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T22:14:41,268 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T22:14:41,269 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T22:14:41,269 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T22:14:41,269 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T22:14:41,269 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T22:14:41,270 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T22:14:41,270 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T22:14:41,270 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T22:14:41,270 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T22:14:41,271 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T22:14:41,271 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T22:14:41,271 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T22:14:41,272 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T22:14:41,272 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T22:14:41,272 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T22:14:41,272 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T22:14:41,272 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T22:14:41,273 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T22:14:41,273 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T22:14:41,273 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T22:14:41,273 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T22:14:41,274 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T22:14:41,274 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T22:14:41,274 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T22:14:41,275 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T22:14:41,275 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T22:14:41,275 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T22:14:41,275 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T22:14:41,275 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T22:14:41,276 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T22:14:41,276 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T22:14:41,276 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T22:14:41,277 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T22:14:41,277 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T22:14:41,277 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T22:14:41,277 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T22:14:41,278 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T22:14:41,278 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T22:14:41,278 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T22:14:41,278 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T22:14:41,279 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T22:14:41,279 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T22:14:41,279 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T22:14:41,279 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T22:14:41,280 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T22:14:41,280 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T22:14:41,280 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T22:14:41,281 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T22:14:41,281 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T22:14:41,281 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T22:14:41,281 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T22:14:41,282 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T22:14:41,282 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T22:14:41,282 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T22:14:41,283 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T22:14:41,283 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T22:14:41,283 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T22:14:41,283 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T22:14:41,284 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T22:14:41,284 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T22:14:41,284 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T22:14:41,284 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T22:14:41,285 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T22:14:41,285 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T22:14:41,285 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T22:14:41,285 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T22:14:41,285 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T22:14:41,286 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T22:14:41,286 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T22:14:41,286 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T22:14:41,286 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T22:14:41,287 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T22:14:41,287 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T22:14:41,287 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T22:14:41,287 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T22:14:41,287 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T22:14:41,288 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T22:14:41,288 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T22:14:41,288 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T22:14:41,289 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T22:14:41,289 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T22:14:41,289 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T22:14:41,289 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T22:14:41,290 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T22:14:41,290 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T22:14:41,290 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T22:14:41,291 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T22:14:41,291 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T22:14:41,291 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T22:14:41,292 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T22:14:41,292 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T22:14:41,292 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T22:14:41,292 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T22:14:41,293 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T22:14:41,293 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T22:14:41,293 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T22:14:41,293 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T22:14:41,293 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T22:14:41,294 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T22:14:41,294 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T22:14:41,294 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T22:14:41,294 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T22:14:41,295 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T22:14:41,295 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T22:14:41,295 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T22:14:41,295 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T22:14:41,296 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T22:14:41,296 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T22:14:41,296 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T22:14:41,297 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T22:14:41,297 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T22:14:41,297 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T22:14:41,297 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T22:14:41,298 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T22:14:41,298 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T22:14:41,298 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T22:14:41,298 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T22:14:41,298 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T22:14:41,299 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T22:14:41,299 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T22:14:41,300 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T22:14:41,300 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T22:14:41,300 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T22:14:41,300 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T22:14:41,300 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T22:14:41,301 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T22:14:41,301 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T22:14:41,301 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T22:14:41,301 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T22:14:41,302 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T22:14:41,302 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T22:14:41,303 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T22:14:41,303 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T22:14:41,303 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T22:14:41,303 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T22:14:41,304 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T22:14:41,304 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T22:14:41,304 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T22:14:41,304 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T22:14:41,304 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T22:14:41,305 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T22:14:41,305 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T22:14:41,305 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T22:14:41,305 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T22:14:41,306 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T22:14:41,306 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T22:14:41,306 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T22:14:41,306 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T22:14:41,307 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T22:14:41,307 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T22:14:41,307 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T22:14:41,307 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T22:14:41,307 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T22:14:41,308 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T22:14:41,308 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T22:14:41,308 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T22:14:41,308 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T22:14:41,309 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T22:14:41,309 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T22:14:41,309 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T22:14:41,309 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T22:14:41,309 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T22:14:41,310 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T22:14:41,310 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T22:14:41,310 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T22:14:41,311 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T22:14:41,311 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T22:14:41,311 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T22:14:41,311 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T22:14:41,312 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T22:14:41,312 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T22:14:41,312 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T22:14:41,313 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T22:14:41,313 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T22:14:41,313 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T22:14:41,313 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T22:14:41,313 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T22:14:41,313 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T22:14:41,314 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T22:14:41,314 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T22:14:41,314 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T22:14:41,314 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T22:14:41,315 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T22:14:41,315 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T22:14:41,315 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T22:14:41,315 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T22:14:41,316 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T22:14:41,316 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T22:14:41,316 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T22:14:41,316 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T22:14:41,317 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T22:14:41,317 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T22:14:41,317 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T22:14:41,317 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T22:14:41,318 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T22:14:41,319 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T22:14:41,319 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T22:14:41,320 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T22:14:41,320 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T22:14:41,321 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T22:14:41,321 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T22:14:41,321 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T22:14:41,321 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T22:14:41,322 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T22:14:41,322 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T22:14:41,322 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T22:14:41,323 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T22:14:41,323 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T22:14:41,323 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T22:14:41,324 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T22:14:41,324 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T22:14:41,324 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T22:14:41,324 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T22:14:41,324 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T22:14:41,325 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T22:14:41,325 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T22:14:41,325 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T22:14:41,325 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T22:14:41,326 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T22:14:41,326 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T22:14:41,326 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T22:14:41,326 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T22:14:41,327 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T22:14:41,327 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T22:14:41,327 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T22:14:41,327 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T22:14:41,327 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T22:14:41,328 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T22:14:41,328 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T22:14:41,328 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T22:14:41,329 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T22:14:41,329 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T22:14:41,329 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T22:14:41,330 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T22:14:41,330 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T22:14:41,330 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T22:14:41,330 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T22:14:41,331 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T22:14:41,331 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T22:14:41,331 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T22:14:41,331 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T22:14:41,332 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T22:14:41,332 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T22:14:41,332 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T22:14:41,332 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T22:14:41,333 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T22:14:41,333 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T22:14:41,333 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T22:14:41,333 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T22:14:41,334 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T22:14:41,334 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T22:14:41,334 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T22:14:41,334 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T22:14:41,335 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T22:14:41,335 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T22:14:41,335 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T22:14:41,336 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T22:14:41,336 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T22:14:41,336 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T22:14:41,336 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T22:14:41,337 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T22:14:41,337 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T22:14:41,337 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T22:14:41,337 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T22:14:41,337 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T22:14:41,338 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T22:14:41,338 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T22:14:41,338 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T22:14:41,338 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T22:14:41,339 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T22:14:41,339 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T22:14:41,340 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T22:14:41,340 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T22:14:41,340 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T22:14:41,341 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T22:14:41,341 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T22:14:41,341 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T22:14:41,341 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T22:14:41,342 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T22:14:41,342 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T22:14:41,342 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T22:14:41,342 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T22:14:41,343 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T22:14:41,343 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T22:14:41,343 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T22:14:41,343 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T22:14:41,344 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T22:14:41,344 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T22:14:41,344 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T22:14:41,344 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T22:14:41,344 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T22:14:41,345 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T22:14:41,345 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T22:14:41,345 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T22:14:41,345 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T22:14:41,345 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T22:14:41,346 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T22:14:41,346 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T22:14:41,346 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T22:14:41,346 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T22:14:41,346 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T22:14:41,347 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T22:14:41,347 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T22:14:41,347 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T22:14:41,347 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T22:14:41,348 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T22:14:41,348 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T22:14:41,348 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T22:14:41,348 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T22:14:41,348 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T22:14:41,349 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T22:14:41,349 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T22:14:41,349 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T22:14:41,350 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T22:14:41,350 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T22:14:41,350 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T22:14:41,350 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T22:14:41,350 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T22:14:41,351 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T22:14:41,351 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T22:14:41,351 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T22:14:41,351 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T22:14:41,351 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T22:14:41,352 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T22:14:41,352 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T22:14:41,352 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T22:14:41,352 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T22:14:41,353 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T22:14:41,353 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T22:14:41,353 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T22:14:41,353 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T22:14:41,354 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T22:14:41,354 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T22:14:41,354 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T22:14:41,354 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T22:14:41,354 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T22:14:41,355 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T22:14:41,355 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T22:14:41,355 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T22:14:41,355 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T22:14:41,355 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T22:14:41,356 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T22:14:41,356 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T22:14:41,356 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T22:14:41,356 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T22:14:41,357 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T22:14:41,357 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T22:14:41,357 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T22:14:41,357 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T22:14:41,357 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T22:14:41,357 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T22:14:41,358 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T22:14:41,358 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T22:14:41,359 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T22:14:41,359 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T22:14:41,359 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T22:14:41,359 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T22:14:41,359 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T22:14:41,359 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T22:14:41,360 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T22:14:41,360 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T22:14:41,360 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T22:14:41,360 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T22:14:41,360 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T22:14:41,361 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T22:14:41,361 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T22:14:41,361 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T22:14:41,361 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T22:14:41,361 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T22:14:41,362 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T22:14:41,362 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T22:14:41,362 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T22:14:41,362 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T22:14:41,362 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T22:14:41,363 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T22:14:41,363 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T22:14:41,363 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T22:14:41,363 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T22:14:41,363 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T22:14:41,364 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T22:14:41,364 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T22:14:41,364 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T22:14:41,364 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T22:14:41,364 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T22:14:41,364 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T22:14:41,365 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T22:14:41,365 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T22:14:41,365 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T22:14:41,365 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T22:14:41,366 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T22:14:41,366 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T22:14:41,366 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T22:14:41,367 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T22:14:41,367 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T22:14:41,367 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T22:14:41,367 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T22:14:41,367 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T22:14:41,367 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T22:14:41,368 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T22:14:41,368 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T22:14:41,368 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T22:14:41,369 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T22:14:41,369 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T22:14:41,369 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T22:14:41,369 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T22:14:41,369 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T22:14:41,370 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T22:14:41,370 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T22:14:41,370 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T22:14:41,370 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T22:14:41,371 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T22:14:41,371 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T22:14:41,371 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T22:14:41,371 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T22:14:41,372 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T22:14:41,372 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T22:14:41,372 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T22:14:41,373 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T22:14:41,373 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T22:14:41,373 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T22:14:41,374 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T22:14:41,374 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T22:14:41,375 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T22:14:41,375 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T22:14:41,375 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T22:14:41,375 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T22:14:41,376 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T22:14:41,376 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T22:14:41,376 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T22:14:41,377 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T22:14:41,377 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T22:14:41,378 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T22:14:41,378 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T22:14:41,379 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T22:14:41,380 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T22:14:41,380 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T22:14:41,381 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T22:14:41,382 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T22:14:41,382 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T22:14:41,383 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T22:14:41,383 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T22:14:41,383 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T22:14:41,384 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T22:14:41,384 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T22:14:41,384 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T22:14:41,384 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T22:14:41,385 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T22:14:41,385 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T22:14:41,386 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T22:14:41,386 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T22:14:41,386 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T22:14:41,386 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T22:14:41,387 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T22:14:41,387 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T22:14:41,387 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T22:14:41,388 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T22:14:41,388 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T22:14:41,388 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T22:14:41,389 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T22:14:41,389 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T22:14:41,389 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T22:14:41,389 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T22:14:41,390 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T22:14:41,390 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T22:14:41,390 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T22:14:41,390 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T22:14:41,391 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T22:14:41,391 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T22:14:41,392 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T22:14:41,392 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T22:14:41,393 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T22:14:41,393 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T22:14:41,394 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T22:14:41,394 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T22:14:41,395 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T22:14:41,395 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T22:14:41,396 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T22:14:41,396 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T22:14:41,397 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T22:14:41,397 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T22:14:41,397 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T22:14:41,397 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T22:14:41,398 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T22:14:41,399 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T22:14:41,399 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T22:14:41,399 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T22:14:41,400 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T22:14:41,400 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T22:14:41,401 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T22:14:41,401 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T22:14:41,401 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T22:14:41,402 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T22:14:41,402 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T22:14:41,402 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T22:14:41,402 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T22:14:41,403 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T22:14:41,403 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T22:14:41,404 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T22:14:41,404 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T22:14:41,404 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T22:14:41,404 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T22:14:41,404 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T22:14:41,405 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T22:14:41,406 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T22:14:41,406 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T22:14:41,407 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T22:14:41,408 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T22:14:41,409 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T22:14:41,410 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T22:14:41,411 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T22:14:41,412 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T22:14:41,412 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T22:14:41,413 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T22:14:41,413 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T22:14:41,414 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T22:14:41,414 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T22:14:41,414 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T22:14:41,415 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T22:14:41,415 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T22:14:41,415 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T22:14:41,415 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T22:14:41,416 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T22:14:41,416 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T22:14:41,416 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T22:14:41,416 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T22:14:41,417 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T22:14:41,417 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T22:14:41,417 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T22:14:41,417 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T22:14:41,418 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T22:14:41,418 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T22:14:41,418 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T22:14:41,419 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T22:14:41,419 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T22:14:41,419 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T22:14:41,419 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T22:14:41,420 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T22:14:41,420 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T22:14:41,420 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T22:14:41,421 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T22:14:41,421 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T22:14:41,422 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T22:14:41,422 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T22:14:41,423 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T22:14:41,423 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T22:14:41,425 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T22:14:41,425 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T22:14:41,428 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T22:14:41,431 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T22:14:41,432 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T22:14:41,433 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T22:14:41,433 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T22:14:41,434 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T22:14:41,435 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T22:14:41,435 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T22:14:41,436 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T22:14:41,437 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T22:14:41,437 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T22:14:41,437 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T22:14:41,438 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T22:14:41,438 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T22:14:41,439 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T22:14:41,439 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T22:14:41,439 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T22:14:41,439 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T22:14:41,440 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T22:14:41,440 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T22:14:41,441 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T22:14:41,441 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T22:14:41,442 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T22:14:41,442 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T22:14:41,442 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T22:14:41,442 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T22:14:41,443 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T22:14:41,443 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T22:14:41,444 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T22:14:41,444 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T22:14:41,444 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T22:14:41,444 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T22:14:41,445 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T22:14:41,445 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T22:14:41,446 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T22:14:41,446 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T22:14:41,447 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T22:14:41,447 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T22:14:41,447 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T22:14:41,448 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T22:14:41,449 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T22:14:41,449 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T22:14:41,449 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T22:14:41,449 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T22:14:41,450 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T22:14:41,450 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T22:14:41,450 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T22:14:41,450 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T22:14:41,451 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T22:14:41,451 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T22:14:41,451 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T22:14:41,452 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T22:14:41,452 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T22:14:41,453 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T22:14:41,453 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T22:14:41,453 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T22:14:41,454 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T22:14:41,454 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T22:14:41,454 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T22:14:41,454 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T22:14:41,454 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T22:14:41,455 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T22:14:41,455 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T22:14:41,455 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T22:14:41,456 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T22:14:41,456 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T22:14:41,456 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T22:14:41,457 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T22:14:41,457 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T22:14:41,457 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T22:14:41,458 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T22:14:41,458 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T22:14:41,458 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T22:14:41,459 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T22:14:41,459 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T22:14:41,460 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T22:14:41,460 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T22:14:41,460 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T22:14:41,461 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T22:14:41,461 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T22:14:41,461 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T22:14:41,462 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T22:14:41,462 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T22:14:41,462 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T22:14:41,463 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T22:14:41,463 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T22:14:41,463 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T22:14:41,464 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T22:14:41,464 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T22:14:41,464 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T22:14:41,464 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T22:14:41,465 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T22:14:41,465 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T22:14:41,465 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T22:14:41,465 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T22:14:41,466 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T22:14:41,466 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T22:14:41,466 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T22:14:41,466 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T22:14:41,467 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T22:14:41,467 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T22:14:41,467 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T22:14:41,467 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T22:14:41,468 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T22:14:41,468 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T22:14:41,469 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T22:14:41,469 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T22:14:41,469 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T22:14:41,470 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T22:14:41,470 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T22:14:41,471 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T22:14:41,471 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T22:14:41,471 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T22:14:41,471 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T22:14:41,471 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T22:14:41,472 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T22:14:41,472 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T22:14:41,472 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T22:14:41,473 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T22:14:41,473 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T22:14:41,473 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T22:14:41,473 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T22:14:41,474 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T22:14:41,474 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T22:14:41,474 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T22:14:41,474 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T22:14:41,475 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T22:14:41,475 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T22:14:41,475 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T22:14:41,476 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T22:14:41,476 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T22:14:41,477 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T22:14:41,477 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T22:14:41,477 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T22:14:41,478 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T22:14:41,478 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T22:14:41,478 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T22:14:41,479 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T22:14:41,479 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T22:14:41,479 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T22:14:41,480 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T22:14:41,480 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T22:14:41,481 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T22:14:41,481 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T22:14:41,482 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T22:14:41,482 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T22:14:41,482 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T22:14:41,483 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T22:14:41,483 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T22:14:41,484 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T22:14:41,484 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T22:14:41,484 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T22:14:41,484 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T22:14:41,485 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T22:14:41,485 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T22:14:41,486 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T22:14:41,486 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T22:14:41,487 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T22:14:41,487 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T22:14:41,487 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T22:14:41,488 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T22:14:41,489 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T22:14:41,490 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T22:14:41,490 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T22:14:41,491 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T22:14:41,492 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T22:14:41,492 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T22:14:41,492 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T22:14:41,492 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T22:14:41,493 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T22:14:41,493 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T22:14:41,493 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T22:14:41,493 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T22:14:41,494 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T22:14:41,494 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T22:14:41,494 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T22:14:41,494 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T22:14:41,495 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T22:14:41,495 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T22:14:41,495 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T22:14:41,495 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T22:14:41,496 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T22:14:41,496 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T22:14:41,496 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T22:14:41,496 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T22:14:41,497 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T22:14:41,497 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T22:14:41,497 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T22:14:41,497 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T22:14:41,497 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T22:14:41,498 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T22:14:41,498 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T22:14:41,498 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T22:14:41,498 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T22:14:41,498 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T22:14:41,499 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T22:14:41,499 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T22:14:41,499 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T22:14:41,499 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T22:14:41,499 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T22:14:41,500 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T22:14:41,500 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T22:14:41,500 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T22:14:41,501 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T22:14:41,501 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T22:14:41,501 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T22:14:41,502 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T22:14:41,502 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T22:14:41,503 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T22:14:41,503 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T22:14:41,503 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T22:14:41,503 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T22:14:41,503 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T22:14:41,503 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T22:14:41,504 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T22:14:41,504 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T22:14:41,504 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T22:14:41,504 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T22:14:41,505 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T22:14:41,505 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T22:14:41,505 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T22:14:41,505 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T22:14:41,505 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T22:14:41,506 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T22:14:41,506 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T22:14:41,506 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T22:14:41,507 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T22:14:41,507 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T22:14:41,507 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T22:14:41,508 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T22:14:41,509 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T22:14:41,509 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T22:14:41,509 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T22:14:41,509 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T22:14:41,510 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T22:14:41,510 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T22:14:41,511 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T22:14:41,511 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T22:14:41,511 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T22:14:41,511 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T22:14:41,512 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T22:14:41,512 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T22:14:41,513 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T22:14:41,513 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T22:14:41,514 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T22:14:41,515 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T22:14:41,515 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T22:14:41,515 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T22:14:41,515 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T22:14:41,516 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T22:14:41,517 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T22:14:41,517 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T22:14:41,518 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T22:14:41,519 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T22:14:41,519 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T22:14:41,520 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T22:14:41,521 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T22:14:41,521 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T22:14:41,522 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T22:14:41,523 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T22:14:41,524 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T22:14:41,524 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T22:14:41,524 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T22:14:41,524 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T22:14:41,525 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T22:14:41,525 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T22:14:41,525 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T22:14:41,526 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T22:14:41,526 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T22:14:41,526 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T22:14:41,527 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T22:14:41,527 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T22:14:41,527 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T22:14:41,528 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T22:14:41,528 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T22:14:41,529 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T22:14:41,529 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T22:14:41,529 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T22:14:41,529 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T22:14:41,529 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T22:14:41,530 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T22:14:41,530 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T22:14:41,530 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T22:14:41,530 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T22:14:41,531 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T22:14:41,531 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T22:14:41,531 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T22:14:41,531 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T22:14:41,532 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T22:14:41,532 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T22:14:41,532 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T22:14:41,532 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T22:14:41,533 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T22:14:41,533 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T22:14:41,533 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T22:14:41,533 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T22:14:41,534 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T22:14:41,534 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T22:14:41,534 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T22:14:41,535 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T22:14:41,535 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T22:14:41,536 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T22:14:41,536 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T22:14:41,536 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T22:14:41,536 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T22:14:41,537 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T22:14:41,537 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T22:14:41,538 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T22:14:41,538 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T22:14:41,538 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T22:14:41,538 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T22:14:41,538 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T22:14:41,538 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T22:14:41,539 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T22:14:41,539 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T22:14:41,539 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T22:14:41,540 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T22:14:41,540 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T22:14:41,541 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T22:14:41,541 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T22:14:41,541 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T22:14:41,542 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T22:14:41,542 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T22:14:41,542 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T22:14:41,542 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T22:14:41,543 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T22:14:41,543 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T22:14:41,543 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T22:14:41,543 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T22:14:41,544 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T22:14:41,544 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T22:14:41,544 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T22:14:41,544 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T22:14:41,545 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T22:14:41,545 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T22:14:41,545 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T22:14:41,545 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T22:14:41,546 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T22:14:41,546 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T22:14:41,546 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T22:14:41,546 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T22:14:41,547 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T22:14:41,547 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T22:14:41,547 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T22:14:41,548 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T22:14:41,548 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T22:14:41,548 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T22:14:41,548 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T22:14:41,549 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T22:14:41,549 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T22:14:41,550 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T22:14:41,550 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T22:14:41,550 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T22:14:41,551 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T22:14:41,551 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T22:14:41,552 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T22:14:41,552 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T22:14:41,552 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T22:14:41,552 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T22:14:41,552 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T22:14:41,553 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T22:14:41,553 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T22:14:41,553 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T22:14:41,554 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T22:14:41,554 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T22:14:41,554 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T22:14:41,554 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T22:14:41,554 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T22:14:41,555 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T22:14:41,555 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T22:14:41,555 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T22:14:41,556 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T22:14:41,556 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T22:14:41,556 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T22:14:41,556 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T22:14:41,556 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T22:14:41,557 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T22:14:41,557 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T22:14:41,557 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T22:14:41,557 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T22:14:41,558 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T22:14:41,558 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T22:14:41,558 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T22:14:41,558 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T22:14:41,559 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T22:14:41,559 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T22:14:41,559 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T22:14:41,559 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T22:14:41,559 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T22:14:41,560 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T22:14:41,560 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T22:14:41,560 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T22:14:41,560 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T22:14:41,561 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T22:14:41,561 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T22:14:41,561 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T22:14:41,561 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T22:14:41,561 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T22:14:41,562 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T22:14:41,562 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T22:14:41,562 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T22:14:41,562 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T22:14:41,562 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T22:14:41,563 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T22:14:41,563 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T22:14:41,563 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T22:14:41,563 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T22:14:41,564 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T22:14:41,564 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T22:14:41,564 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T22:14:41,564 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T22:14:41,565 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T22:14:41,565 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T22:14:41,565 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T22:14:41,565 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T22:14:41,565 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T22:14:41,565 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T22:14:41,566 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T22:14:41,566 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T22:14:41,566 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T22:14:41,566 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T22:14:41,567 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T22:14:41,567 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T22:14:41,567 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T22:14:41,568 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T22:14:41,568 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T22:14:41,569 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T22:14:41,569 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T22:14:41,569 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T22:14:41,570 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T22:14:41,570 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T22:14:41,570 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T22:14:41,571 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T22:14:41,571 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T22:14:41,572 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T22:14:41,572 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T22:14:41,573 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T22:14:41,576 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T22:14:41,577 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40087 {}] regionserver.HRegion(8855): Flush requested on 63842f73863e9e7b9587486b122e6aae 2024-11-24T22:14:41,577 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 63842f73863e9e7b9587486b122e6aae 3/3 column families, dataSize=46.96 KB heapSize=140.20 KB 2024-11-24T22:14:41,578 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 63842f73863e9e7b9587486b122e6aae, store=A 2024-11-24T22:14:41,578 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-24T22:14:41,578 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 63842f73863e9e7b9587486b122e6aae, store=B 2024-11-24T22:14:41,578 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-24T22:14:41,578 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 63842f73863e9e7b9587486b122e6aae, store=C 2024-11-24T22:14:41,578 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-24T22:14:41,587 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411247610d45b6bd7401eb3c8ff9a83fd52f9_63842f73863e9e7b9587486b122e6aae is 50, key is test_row_0/A:col10/1732486481512/Put/seqid=0 2024-11-24T22:14:41,593 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45929 is added to blk_1073742039_1215 (size=14994) 2024-11-24T22:14:41,594 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T22:14:41,599 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411247610d45b6bd7401eb3c8ff9a83fd52f9_63842f73863e9e7b9587486b122e6aae to hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411247610d45b6bd7401eb3c8ff9a83fd52f9_63842f73863e9e7b9587486b122e6aae 2024-11-24T22:14:41,600 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/63842f73863e9e7b9587486b122e6aae/.tmp/A/dd2be7ae7d4045a881cc178f9e22441d, store: [table=TestAcidGuarantees family=A region=63842f73863e9e7b9587486b122e6aae] 2024-11-24T22:14:41,601 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/63842f73863e9e7b9587486b122e6aae/.tmp/A/dd2be7ae7d4045a881cc178f9e22441d is 175, key is test_row_0/A:col10/1732486481512/Put/seqid=0 2024-11-24T22:14:41,604 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45929 is added to blk_1073742040_1216 (size=39949) 2024-11-24T22:14:41,605 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=381, memsize=15.7 K, hasBloomFilter=true, into tmp file hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/63842f73863e9e7b9587486b122e6aae/.tmp/A/dd2be7ae7d4045a881cc178f9e22441d 2024-11-24T22:14:41,614 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/63842f73863e9e7b9587486b122e6aae/.tmp/B/1e6412e76d09485b8b41413a341073af is 50, key is test_row_0/B:col10/1732486481512/Put/seqid=0 2024-11-24T22:14:41,616 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/bd6592c37bb2:0-1 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=309, cellSize=14145, totalHeapSize=32376, min timestamp=1732486481590, max timestamp=1732486481613Num uniques 859; 2024-11-24T22:14:41,616 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/bd6592c37bb2:0-1 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=14145, suffix heap size=117064,new segment heap size=32376 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=309 2024-11-24T22:14:41,620 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/bd6592c37bb2:0-7 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=309, cellSize=14145, totalHeapSize=32376, min timestamp=1732486481590, max timestamp=1732486481613Num uniques 859; 2024-11-24T22:14:41,620 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/bd6592c37bb2:0-7 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=14145, suffix heap size=117064,new segment heap size=32376 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=309 2024-11-24T22:14:41,620 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/bd6592c37bb2:0-0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=309, cellSize=14145, totalHeapSize=32376, min timestamp=1732486481590, max timestamp=1732486481613Num uniques 859; 2024-11-24T22:14:41,620 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/bd6592c37bb2:0-0 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=14145, suffix heap size=117064,new segment heap size=32376 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=309 2024-11-24T22:14:41,622 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45929 is added to blk_1073742041_1217 (size=12301) 2024-11-24T22:14:41,622 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=381 (bloomFilter=true), to=hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/63842f73863e9e7b9587486b122e6aae/.tmp/B/1e6412e76d09485b8b41413a341073af 2024-11-24T22:14:41,630 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/63842f73863e9e7b9587486b122e6aae/.tmp/C/1730f0d31ed04656b27dbffd7e1537c6 is 50, key is test_row_0/C:col10/1732486481512/Put/seqid=0 2024-11-24T22:14:41,634 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45929 is added to blk_1073742042_1218 (size=12301) 2024-11-24T22:14:41,634 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=381 (bloomFilter=true), to=hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/63842f73863e9e7b9587486b122e6aae/.tmp/C/1730f0d31ed04656b27dbffd7e1537c6 2024-11-24T22:14:41,639 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/63842f73863e9e7b9587486b122e6aae/.tmp/A/dd2be7ae7d4045a881cc178f9e22441d as hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/63842f73863e9e7b9587486b122e6aae/A/dd2be7ae7d4045a881cc178f9e22441d 2024-11-24T22:14:41,643 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/63842f73863e9e7b9587486b122e6aae/A/dd2be7ae7d4045a881cc178f9e22441d, entries=200, sequenceid=381, filesize=39.0 K 2024-11-24T22:14:41,644 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/63842f73863e9e7b9587486b122e6aae/.tmp/B/1e6412e76d09485b8b41413a341073af as hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/63842f73863e9e7b9587486b122e6aae/B/1e6412e76d09485b8b41413a341073af 2024-11-24T22:14:41,648 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/63842f73863e9e7b9587486b122e6aae/B/1e6412e76d09485b8b41413a341073af, entries=150, sequenceid=381, filesize=12.0 K 2024-11-24T22:14:41,649 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/63842f73863e9e7b9587486b122e6aae/.tmp/C/1730f0d31ed04656b27dbffd7e1537c6 as hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/63842f73863e9e7b9587486b122e6aae/C/1730f0d31ed04656b27dbffd7e1537c6 2024-11-24T22:14:41,653 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40087 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=63842f73863e9e7b9587486b122e6aae, server=bd6592c37bb2,40087,1732486423713 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-24T22:14:41,653 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40087 {}] ipc.CallRunner(138): callId: 89 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40202 deadline: 1732486541647, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=63842f73863e9e7b9587486b122e6aae, server=bd6592c37bb2,40087,1732486423713 2024-11-24T22:14:41,653 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40087 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=63842f73863e9e7b9587486b122e6aae, server=bd6592c37bb2,40087,1732486423713 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-24T22:14:41,653 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40087 {}] ipc.CallRunner(138): callId: 176 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40206 deadline: 1732486541647, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=63842f73863e9e7b9587486b122e6aae, server=bd6592c37bb2,40087,1732486423713 2024-11-24T22:14:41,654 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40087 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=63842f73863e9e7b9587486b122e6aae, server=bd6592c37bb2,40087,1732486423713 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-24T22:14:41,654 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40087 {}] ipc.CallRunner(138): callId: 164 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40244 deadline: 1732486541649, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=63842f73863e9e7b9587486b122e6aae, server=bd6592c37bb2,40087,1732486423713 2024-11-24T22:14:41,654 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732486460633.63842f73863e9e7b9587486b122e6aae., hostname=bd6592c37bb2,40087,1732486423713, seqNum=5 , the old value is region=TestAcidGuarantees,,1732486460633.63842f73863e9e7b9587486b122e6aae., hostname=bd6592c37bb2,40087,1732486423713, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=63842f73863e9e7b9587486b122e6aae, server=bd6592c37bb2,40087,1732486423713 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T22:14:41,654 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732486460633.63842f73863e9e7b9587486b122e6aae., hostname=bd6592c37bb2,40087,1732486423713, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=63842f73863e9e7b9587486b122e6aae, server=bd6592c37bb2,40087,1732486423713 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T22:14:41,655 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732486460633.63842f73863e9e7b9587486b122e6aae., hostname=bd6592c37bb2,40087,1732486423713, seqNum=5 because the exception is null or not the one we care about 2024-11-24T22:14:41,655 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732486460633.63842f73863e9e7b9587486b122e6aae., hostname=bd6592c37bb2,40087,1732486423713, seqNum=5 , the old value is region=TestAcidGuarantees,,1732486460633.63842f73863e9e7b9587486b122e6aae., hostname=bd6592c37bb2,40087,1732486423713, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=63842f73863e9e7b9587486b122e6aae, server=bd6592c37bb2,40087,1732486423713 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T22:14:41,656 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732486460633.63842f73863e9e7b9587486b122e6aae., hostname=bd6592c37bb2,40087,1732486423713, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=63842f73863e9e7b9587486b122e6aae, server=bd6592c37bb2,40087,1732486423713 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T22:14:41,656 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732486460633.63842f73863e9e7b9587486b122e6aae., hostname=bd6592c37bb2,40087,1732486423713, seqNum=5 because the exception is null or not the one we care about 2024-11-24T22:14:41,657 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732486460633.63842f73863e9e7b9587486b122e6aae., hostname=bd6592c37bb2,40087,1732486423713, seqNum=5 , the old value is region=TestAcidGuarantees,,1732486460633.63842f73863e9e7b9587486b122e6aae., hostname=bd6592c37bb2,40087,1732486423713, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=63842f73863e9e7b9587486b122e6aae, server=bd6592c37bb2,40087,1732486423713 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T22:14:41,657 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732486460633.63842f73863e9e7b9587486b122e6aae., hostname=bd6592c37bb2,40087,1732486423713, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=63842f73863e9e7b9587486b122e6aae, server=bd6592c37bb2,40087,1732486423713 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T22:14:41,657 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732486460633.63842f73863e9e7b9587486b122e6aae., hostname=bd6592c37bb2,40087,1732486423713, seqNum=5 because the exception is null or not the one we care about 2024-11-24T22:14:41,657 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/63842f73863e9e7b9587486b122e6aae/C/1730f0d31ed04656b27dbffd7e1537c6, entries=150, sequenceid=381, filesize=12.0 K 2024-11-24T22:14:41,658 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~46.96 KB/48090, heapSize ~140.16 KB/143520, currentSize=140.89 KB/144270 for 63842f73863e9e7b9587486b122e6aae in 81ms, sequenceid=381, compaction requested=true 2024-11-24T22:14:41,658 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 63842f73863e9e7b9587486b122e6aae: 2024-11-24T22:14:41,659 DEBUG [RS:0;bd6592c37bb2:40087-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-24T22:14:41,659 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 63842f73863e9e7b9587486b122e6aae:A, priority=-2147483648, current under compaction store size is 1 2024-11-24T22:14:41,659 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-24T22:14:41,659 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 63842f73863e9e7b9587486b122e6aae:B, priority=-2147483648, current under compaction store size is 2 2024-11-24T22:14:41,659 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-24T22:14:41,659 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 63842f73863e9e7b9587486b122e6aae:C, priority=-2147483648, current under compaction store size is 3 2024-11-24T22:14:41,659 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=2), splitQueue=0 2024-11-24T22:14:41,659 DEBUG [RS:0;bd6592c37bb2:40087-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-24T22:14:41,660 DEBUG [RS:0;bd6592c37bb2:40087-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 103243 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-24T22:14:41,661 DEBUG [RS:0;bd6592c37bb2:40087-longCompactions-0 {}] regionserver.HStore(1541): 63842f73863e9e7b9587486b122e6aae/A is initiating minor compaction (all files) 2024-11-24T22:14:41,661 INFO [RS:0;bd6592c37bb2:40087-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 63842f73863e9e7b9587486b122e6aae/A in TestAcidGuarantees,,1732486460633.63842f73863e9e7b9587486b122e6aae. 2024-11-24T22:14:41,661 INFO [RS:0;bd6592c37bb2:40087-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/63842f73863e9e7b9587486b122e6aae/A/477957c1db11458f8b66d226b8ee3aeb, hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/63842f73863e9e7b9587486b122e6aae/A/b45983b335d44ecb8c6b81e63ccb78e4, hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/63842f73863e9e7b9587486b122e6aae/A/dd2be7ae7d4045a881cc178f9e22441d] into tmpdir=hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/63842f73863e9e7b9587486b122e6aae/.tmp, totalSize=100.8 K 2024-11-24T22:14:41,661 INFO [RS:0;bd6592c37bb2:40087-longCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=13 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1732486460633.63842f73863e9e7b9587486b122e6aae. 2024-11-24T22:14:41,661 DEBUG [RS:0;bd6592c37bb2:40087-longCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1732486460633.63842f73863e9e7b9587486b122e6aae. files: [hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/63842f73863e9e7b9587486b122e6aae/A/477957c1db11458f8b66d226b8ee3aeb, hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/63842f73863e9e7b9587486b122e6aae/A/b45983b335d44ecb8c6b81e63ccb78e4, hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/63842f73863e9e7b9587486b122e6aae/A/dd2be7ae7d4045a881cc178f9e22441d] 2024-11-24T22:14:41,661 DEBUG [RS:0;bd6592c37bb2:40087-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37687 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-24T22:14:41,661 DEBUG [RS:0;bd6592c37bb2:40087-shortCompactions-0 {}] regionserver.HStore(1541): 63842f73863e9e7b9587486b122e6aae/B is initiating minor compaction (all files) 2024-11-24T22:14:41,661 INFO [RS:0;bd6592c37bb2:40087-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 63842f73863e9e7b9587486b122e6aae/B in TestAcidGuarantees,,1732486460633.63842f73863e9e7b9587486b122e6aae. 2024-11-24T22:14:41,661 INFO [RS:0;bd6592c37bb2:40087-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/63842f73863e9e7b9587486b122e6aae/B/48419645ce0442d981fa746ee68828e1, hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/63842f73863e9e7b9587486b122e6aae/B/916c780a9f1c48a7a780b2aa308cc2e3, hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/63842f73863e9e7b9587486b122e6aae/B/1e6412e76d09485b8b41413a341073af] into tmpdir=hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/63842f73863e9e7b9587486b122e6aae/.tmp, totalSize=36.8 K 2024-11-24T22:14:41,662 DEBUG [RS:0;bd6592c37bb2:40087-longCompactions-0 {}] compactions.Compactor(225): Compacting 477957c1db11458f8b66d226b8ee3aeb, keycount=150, bloomtype=ROW, size=31.3 K, encoding=NONE, compression=NONE, seqNum=357, earliestPutTs=1732486479160 2024-11-24T22:14:41,662 DEBUG [RS:0;bd6592c37bb2:40087-shortCompactions-0 {}] compactions.Compactor(225): Compacting 48419645ce0442d981fa746ee68828e1, keycount=150, bloomtype=ROW, size=12.8 K, encoding=NONE, compression=NONE, seqNum=357, earliestPutTs=1732486479160 2024-11-24T22:14:41,662 DEBUG [RS:0;bd6592c37bb2:40087-shortCompactions-0 {}] compactions.Compactor(225): Compacting 916c780a9f1c48a7a780b2aa308cc2e3, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=368, earliestPutTs=1732486479302 2024-11-24T22:14:41,663 DEBUG [RS:0;bd6592c37bb2:40087-longCompactions-0 {}] compactions.Compactor(225): Compacting b45983b335d44ecb8c6b81e63ccb78e4, keycount=150, bloomtype=ROW, size=30.5 K, encoding=NONE, compression=NONE, seqNum=368, earliestPutTs=1732486479302 2024-11-24T22:14:41,669 DEBUG [RS:0;bd6592c37bb2:40087-longCompactions-0 {}] compactions.Compactor(225): Compacting dd2be7ae7d4045a881cc178f9e22441d, keycount=200, bloomtype=ROW, size=39.0 K, encoding=NONE, compression=NONE, seqNum=381, earliestPutTs=1732486481512 2024-11-24T22:14:41,669 DEBUG [RS:0;bd6592c37bb2:40087-shortCompactions-0 {}] compactions.Compactor(225): Compacting 1e6412e76d09485b8b41413a341073af, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=381, earliestPutTs=1732486481512 2024-11-24T22:14:41,681 INFO [RS:0;bd6592c37bb2:40087-longCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=63842f73863e9e7b9587486b122e6aae] 2024-11-24T22:14:41,696 INFO [RS:0;bd6592c37bb2:40087-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 63842f73863e9e7b9587486b122e6aae#B#compaction#178 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-24T22:14:41,697 DEBUG [RS:0;bd6592c37bb2:40087-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/63842f73863e9e7b9587486b122e6aae/.tmp/B/0be02238389c498580e6164758290ed8 is 50, key is test_row_0/B:col10/1732486481512/Put/seqid=0 2024-11-24T22:14:41,699 DEBUG [RS:0;bd6592c37bb2:40087-longCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e20241124ace1b6bb7c05456b935f6810ad437788_63842f73863e9e7b9587486b122e6aae store=[table=TestAcidGuarantees family=A region=63842f73863e9e7b9587486b122e6aae] 2024-11-24T22:14:41,701 DEBUG [RS:0;bd6592c37bb2:40087-longCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e20241124ace1b6bb7c05456b935f6810ad437788_63842f73863e9e7b9587486b122e6aae, store=[table=TestAcidGuarantees family=A region=63842f73863e9e7b9587486b122e6aae] 2024-11-24T22:14:41,701 DEBUG [RS:0;bd6592c37bb2:40087-longCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241124ace1b6bb7c05456b935f6810ad437788_63842f73863e9e7b9587486b122e6aae because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=63842f73863e9e7b9587486b122e6aae] 2024-11-24T22:14:41,714 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45929 is added to blk_1073742043_1219 (size=13187) 2024-11-24T22:14:41,722 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45929 is added to blk_1073742044_1220 (size=4469) 2024-11-24T22:14:41,726 INFO [RS:0;bd6592c37bb2:40087-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 63842f73863e9e7b9587486b122e6aae#A#compaction#177 average throughput is 0.54 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-24T22:14:41,726 DEBUG [RS:0;bd6592c37bb2:40087-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/63842f73863e9e7b9587486b122e6aae/.tmp/A/2b8c21ff057c4163b79d571652f1f38c is 175, key is test_row_0/A:col10/1732486481512/Put/seqid=0 2024-11-24T22:14:41,734 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45929 is added to blk_1073742045_1221 (size=32141) 2024-11-24T22:14:41,763 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40087 {}] regionserver.HRegion(8855): Flush requested on 63842f73863e9e7b9587486b122e6aae 2024-11-24T22:14:41,763 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 63842f73863e9e7b9587486b122e6aae 3/3 column families, dataSize=147.60 KB heapSize=410.77 KB 2024-11-24T22:14:41,764 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 63842f73863e9e7b9587486b122e6aae, store=A 2024-11-24T22:14:41,764 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-24T22:14:41,764 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 63842f73863e9e7b9587486b122e6aae, store=B 2024-11-24T22:14:41,764 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-24T22:14:41,764 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 63842f73863e9e7b9587486b122e6aae, store=C 2024-11-24T22:14:41,764 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-24T22:14:41,771 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024112466703d3eec03428db38bbc64c672f9c9_63842f73863e9e7b9587486b122e6aae is 50, key is test_row_0/A:col10/1732486481761/Put/seqid=0 2024-11-24T22:14:41,777 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45929 is added to blk_1073742046_1222 (size=14994) 2024-11-24T22:14:41,780 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40087 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=63842f73863e9e7b9587486b122e6aae, server=bd6592c37bb2,40087,1732486423713 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-24T22:14:41,781 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40087 {}] ipc.CallRunner(138): callId: 167 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40244 deadline: 1732486541774, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=63842f73863e9e7b9587486b122e6aae, server=bd6592c37bb2,40087,1732486423713 2024-11-24T22:14:41,781 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732486460633.63842f73863e9e7b9587486b122e6aae., hostname=bd6592c37bb2,40087,1732486423713, seqNum=5 , the old value is region=TestAcidGuarantees,,1732486460633.63842f73863e9e7b9587486b122e6aae., hostname=bd6592c37bb2,40087,1732486423713, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=63842f73863e9e7b9587486b122e6aae, server=bd6592c37bb2,40087,1732486423713 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T22:14:41,781 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732486460633.63842f73863e9e7b9587486b122e6aae., hostname=bd6592c37bb2,40087,1732486423713, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=63842f73863e9e7b9587486b122e6aae, server=bd6592c37bb2,40087,1732486423713 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T22:14:41,781 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732486460633.63842f73863e9e7b9587486b122e6aae., hostname=bd6592c37bb2,40087,1732486423713, seqNum=5 because the exception is null or not the one we care about 2024-11-24T22:14:41,785 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40087 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=63842f73863e9e7b9587486b122e6aae, server=bd6592c37bb2,40087,1732486423713 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-24T22:14:41,786 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40087 {}] ipc.CallRunner(138): callId: 179 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40206 deadline: 1732486541781, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=63842f73863e9e7b9587486b122e6aae, server=bd6592c37bb2,40087,1732486423713 2024-11-24T22:14:41,786 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40087 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=63842f73863e9e7b9587486b122e6aae, server=bd6592c37bb2,40087,1732486423713 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-24T22:14:41,786 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40087 {}] ipc.CallRunner(138): callId: 93 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40202 deadline: 1732486541781, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=63842f73863e9e7b9587486b122e6aae, server=bd6592c37bb2,40087,1732486423713 2024-11-24T22:14:41,786 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732486460633.63842f73863e9e7b9587486b122e6aae., hostname=bd6592c37bb2,40087,1732486423713, seqNum=5 , the old value is region=TestAcidGuarantees,,1732486460633.63842f73863e9e7b9587486b122e6aae., hostname=bd6592c37bb2,40087,1732486423713, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=63842f73863e9e7b9587486b122e6aae, server=bd6592c37bb2,40087,1732486423713 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T22:14:41,786 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732486460633.63842f73863e9e7b9587486b122e6aae., hostname=bd6592c37bb2,40087,1732486423713, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=63842f73863e9e7b9587486b122e6aae, server=bd6592c37bb2,40087,1732486423713 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T22:14:41,786 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732486460633.63842f73863e9e7b9587486b122e6aae., hostname=bd6592c37bb2,40087,1732486423713, seqNum=5 because the exception is null or not the one we care about 2024-11-24T22:14:41,786 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732486460633.63842f73863e9e7b9587486b122e6aae., hostname=bd6592c37bb2,40087,1732486423713, seqNum=5 , the old value is region=TestAcidGuarantees,,1732486460633.63842f73863e9e7b9587486b122e6aae., hostname=bd6592c37bb2,40087,1732486423713, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=63842f73863e9e7b9587486b122e6aae, server=bd6592c37bb2,40087,1732486423713 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T22:14:41,786 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732486460633.63842f73863e9e7b9587486b122e6aae., hostname=bd6592c37bb2,40087,1732486423713, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=63842f73863e9e7b9587486b122e6aae, server=bd6592c37bb2,40087,1732486423713 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T22:14:41,786 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732486460633.63842f73863e9e7b9587486b122e6aae., hostname=bd6592c37bb2,40087,1732486423713, seqNum=5 because the exception is null or not the one we care about 2024-11-24T22:14:41,891 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40087 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=63842f73863e9e7b9587486b122e6aae, server=bd6592c37bb2,40087,1732486423713 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-24T22:14:41,891 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40087 {}] ipc.CallRunner(138): callId: 168 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40244 deadline: 1732486541891, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=63842f73863e9e7b9587486b122e6aae, server=bd6592c37bb2,40087,1732486423713 2024-11-24T22:14:41,891 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40087 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=63842f73863e9e7b9587486b122e6aae, server=bd6592c37bb2,40087,1732486423713 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-24T22:14:41,892 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732486460633.63842f73863e9e7b9587486b122e6aae., hostname=bd6592c37bb2,40087,1732486423713, seqNum=5 , the old value is region=TestAcidGuarantees,,1732486460633.63842f73863e9e7b9587486b122e6aae., hostname=bd6592c37bb2,40087,1732486423713, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=63842f73863e9e7b9587486b122e6aae, server=bd6592c37bb2,40087,1732486423713 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T22:14:41,892 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40087 {}] ipc.CallRunner(138): callId: 180 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40206 deadline: 1732486541891, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=63842f73863e9e7b9587486b122e6aae, server=bd6592c37bb2,40087,1732486423713 2024-11-24T22:14:41,891 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40087 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=63842f73863e9e7b9587486b122e6aae, server=bd6592c37bb2,40087,1732486423713 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-24T22:14:41,892 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732486460633.63842f73863e9e7b9587486b122e6aae., hostname=bd6592c37bb2,40087,1732486423713, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=63842f73863e9e7b9587486b122e6aae, server=bd6592c37bb2,40087,1732486423713 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T22:14:41,892 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732486460633.63842f73863e9e7b9587486b122e6aae., hostname=bd6592c37bb2,40087,1732486423713, seqNum=5 because the exception is null or not the one we care about 2024-11-24T22:14:41,892 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40087 {}] ipc.CallRunner(138): callId: 94 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40202 deadline: 1732486541891, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=63842f73863e9e7b9587486b122e6aae, server=bd6592c37bb2,40087,1732486423713 2024-11-24T22:14:41,892 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732486460633.63842f73863e9e7b9587486b122e6aae., hostname=bd6592c37bb2,40087,1732486423713, seqNum=5 , the old value is region=TestAcidGuarantees,,1732486460633.63842f73863e9e7b9587486b122e6aae., hostname=bd6592c37bb2,40087,1732486423713, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=63842f73863e9e7b9587486b122e6aae, server=bd6592c37bb2,40087,1732486423713 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T22:14:41,892 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732486460633.63842f73863e9e7b9587486b122e6aae., hostname=bd6592c37bb2,40087,1732486423713, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=63842f73863e9e7b9587486b122e6aae, server=bd6592c37bb2,40087,1732486423713 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T22:14:41,892 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732486460633.63842f73863e9e7b9587486b122e6aae., hostname=bd6592c37bb2,40087,1732486423713, seqNum=5 because the exception is null or not the one we care about 2024-11-24T22:14:41,892 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732486460633.63842f73863e9e7b9587486b122e6aae., hostname=bd6592c37bb2,40087,1732486423713, seqNum=5 , the old value is region=TestAcidGuarantees,,1732486460633.63842f73863e9e7b9587486b122e6aae., hostname=bd6592c37bb2,40087,1732486423713, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=63842f73863e9e7b9587486b122e6aae, server=bd6592c37bb2,40087,1732486423713 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T22:14:41,892 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732486460633.63842f73863e9e7b9587486b122e6aae., hostname=bd6592c37bb2,40087,1732486423713, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=63842f73863e9e7b9587486b122e6aae, server=bd6592c37bb2,40087,1732486423713 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T22:14:41,892 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732486460633.63842f73863e9e7b9587486b122e6aae., hostname=bd6592c37bb2,40087,1732486423713, seqNum=5 because the exception is null or not the one we care about 2024-11-24T22:14:42,015 DEBUG [FsDatasetAsyncDiskServiceFixer {}] hbase.HBaseTestingUtil$FsDatasetAsyncDiskServiceFixer(597): NoSuchFieldException: threadGroup; It might because your Hadoop version > 3.2.3 or 3.3.4, See HBASE-27595 for details. 2024-11-24T22:14:42,104 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40087 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=63842f73863e9e7b9587486b122e6aae, server=bd6592c37bb2,40087,1732486423713 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-24T22:14:42,104 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40087 {}] ipc.CallRunner(138): callId: 169 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40244 deadline: 1732486542100, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=63842f73863e9e7b9587486b122e6aae, server=bd6592c37bb2,40087,1732486423713 2024-11-24T22:14:42,105 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40087 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=63842f73863e9e7b9587486b122e6aae, server=bd6592c37bb2,40087,1732486423713 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-24T22:14:42,105 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732486460633.63842f73863e9e7b9587486b122e6aae., hostname=bd6592c37bb2,40087,1732486423713, seqNum=5 , the old value is region=TestAcidGuarantees,,1732486460633.63842f73863e9e7b9587486b122e6aae., hostname=bd6592c37bb2,40087,1732486423713, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=63842f73863e9e7b9587486b122e6aae, server=bd6592c37bb2,40087,1732486423713 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T22:14:42,105 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732486460633.63842f73863e9e7b9587486b122e6aae., hostname=bd6592c37bb2,40087,1732486423713, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=63842f73863e9e7b9587486b122e6aae, server=bd6592c37bb2,40087,1732486423713 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T22:14:42,105 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40087 {}] ipc.CallRunner(138): callId: 181 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40206 deadline: 1732486542101, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=63842f73863e9e7b9587486b122e6aae, server=bd6592c37bb2,40087,1732486423713 2024-11-24T22:14:42,105 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732486460633.63842f73863e9e7b9587486b122e6aae., hostname=bd6592c37bb2,40087,1732486423713, seqNum=5 because the exception is null or not the one we care about 2024-11-24T22:14:42,105 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40087 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=63842f73863e9e7b9587486b122e6aae, server=bd6592c37bb2,40087,1732486423713 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-24T22:14:42,105 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40087 {}] ipc.CallRunner(138): callId: 95 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40202 deadline: 1732486542101, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=63842f73863e9e7b9587486b122e6aae, server=bd6592c37bb2,40087,1732486423713 2024-11-24T22:14:42,105 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732486460633.63842f73863e9e7b9587486b122e6aae., hostname=bd6592c37bb2,40087,1732486423713, seqNum=5 , the old value is region=TestAcidGuarantees,,1732486460633.63842f73863e9e7b9587486b122e6aae., hostname=bd6592c37bb2,40087,1732486423713, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=63842f73863e9e7b9587486b122e6aae, server=bd6592c37bb2,40087,1732486423713 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T22:14:42,105 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732486460633.63842f73863e9e7b9587486b122e6aae., hostname=bd6592c37bb2,40087,1732486423713, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=63842f73863e9e7b9587486b122e6aae, server=bd6592c37bb2,40087,1732486423713 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T22:14:42,105 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732486460633.63842f73863e9e7b9587486b122e6aae., hostname=bd6592c37bb2,40087,1732486423713, seqNum=5 because the exception is null or not the one we care about 2024-11-24T22:14:42,105 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732486460633.63842f73863e9e7b9587486b122e6aae., hostname=bd6592c37bb2,40087,1732486423713, seqNum=5 , the old value is region=TestAcidGuarantees,,1732486460633.63842f73863e9e7b9587486b122e6aae., hostname=bd6592c37bb2,40087,1732486423713, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=63842f73863e9e7b9587486b122e6aae, server=bd6592c37bb2,40087,1732486423713 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T22:14:42,105 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732486460633.63842f73863e9e7b9587486b122e6aae., hostname=bd6592c37bb2,40087,1732486423713, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=63842f73863e9e7b9587486b122e6aae, server=bd6592c37bb2,40087,1732486423713 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T22:14:42,105 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732486460633.63842f73863e9e7b9587486b122e6aae., hostname=bd6592c37bb2,40087,1732486423713, seqNum=5 because the exception is null or not the one we care about 2024-11-24T22:14:42,120 DEBUG [RS:0;bd6592c37bb2:40087-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/63842f73863e9e7b9587486b122e6aae/.tmp/B/0be02238389c498580e6164758290ed8 as hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/63842f73863e9e7b9587486b122e6aae/B/0be02238389c498580e6164758290ed8 2024-11-24T22:14:42,127 INFO [RS:0;bd6592c37bb2:40087-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 63842f73863e9e7b9587486b122e6aae/B of 63842f73863e9e7b9587486b122e6aae into 0be02238389c498580e6164758290ed8(size=12.9 K), total size for store is 12.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-24T22:14:42,127 DEBUG [RS:0;bd6592c37bb2:40087-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 63842f73863e9e7b9587486b122e6aae: 2024-11-24T22:14:42,127 INFO [RS:0;bd6592c37bb2:40087-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732486460633.63842f73863e9e7b9587486b122e6aae., storeName=63842f73863e9e7b9587486b122e6aae/B, priority=13, startTime=1732486481659; duration=0sec 2024-11-24T22:14:42,127 DEBUG [RS:0;bd6592c37bb2:40087-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-24T22:14:42,127 DEBUG [RS:0;bd6592c37bb2:40087-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 63842f73863e9e7b9587486b122e6aae:B 2024-11-24T22:14:42,127 DEBUG [RS:0;bd6592c37bb2:40087-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-24T22:14:42,128 DEBUG [RS:0;bd6592c37bb2:40087-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37687 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-24T22:14:42,128 DEBUG [RS:0;bd6592c37bb2:40087-shortCompactions-0 {}] regionserver.HStore(1541): 63842f73863e9e7b9587486b122e6aae/C is initiating minor compaction (all files) 2024-11-24T22:14:42,128 INFO [RS:0;bd6592c37bb2:40087-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 63842f73863e9e7b9587486b122e6aae/C in TestAcidGuarantees,,1732486460633.63842f73863e9e7b9587486b122e6aae. 2024-11-24T22:14:42,128 INFO [RS:0;bd6592c37bb2:40087-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/63842f73863e9e7b9587486b122e6aae/C/e2065151f0e24dcdada181dbbfc3e7f8, hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/63842f73863e9e7b9587486b122e6aae/C/f0e1b796e92c45918e07acbffa3c8650, hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/63842f73863e9e7b9587486b122e6aae/C/1730f0d31ed04656b27dbffd7e1537c6] into tmpdir=hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/63842f73863e9e7b9587486b122e6aae/.tmp, totalSize=36.8 K 2024-11-24T22:14:42,130 DEBUG [RS:0;bd6592c37bb2:40087-shortCompactions-0 {}] compactions.Compactor(225): Compacting e2065151f0e24dcdada181dbbfc3e7f8, keycount=150, bloomtype=ROW, size=12.8 K, encoding=NONE, compression=NONE, seqNum=357, earliestPutTs=1732486479160 2024-11-24T22:14:42,130 DEBUG [RS:0;bd6592c37bb2:40087-shortCompactions-0 {}] compactions.Compactor(225): Compacting f0e1b796e92c45918e07acbffa3c8650, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=368, earliestPutTs=1732486479302 2024-11-24T22:14:42,130 DEBUG [RS:0;bd6592c37bb2:40087-shortCompactions-0 {}] compactions.Compactor(225): Compacting 1730f0d31ed04656b27dbffd7e1537c6, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=381, earliestPutTs=1732486481512 2024-11-24T22:14:42,141 INFO [RS:0;bd6592c37bb2:40087-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 63842f73863e9e7b9587486b122e6aae#C#compaction#180 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-24T22:14:42,141 DEBUG [RS:0;bd6592c37bb2:40087-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/63842f73863e9e7b9587486b122e6aae/.tmp/C/0502b36d90f4444c9da4fb69ebf63c97 is 50, key is test_row_0/C:col10/1732486481512/Put/seqid=0 2024-11-24T22:14:42,144 DEBUG [RS:0;bd6592c37bb2:40087-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/63842f73863e9e7b9587486b122e6aae/.tmp/A/2b8c21ff057c4163b79d571652f1f38c as hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/63842f73863e9e7b9587486b122e6aae/A/2b8c21ff057c4163b79d571652f1f38c 2024-11-24T22:14:42,149 INFO [RS:0;bd6592c37bb2:40087-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 63842f73863e9e7b9587486b122e6aae/A of 63842f73863e9e7b9587486b122e6aae into 2b8c21ff057c4163b79d571652f1f38c(size=31.4 K), total size for store is 31.4 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-24T22:14:42,149 DEBUG [RS:0;bd6592c37bb2:40087-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 63842f73863e9e7b9587486b122e6aae: 2024-11-24T22:14:42,149 INFO [RS:0;bd6592c37bb2:40087-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732486460633.63842f73863e9e7b9587486b122e6aae., storeName=63842f73863e9e7b9587486b122e6aae/A, priority=13, startTime=1732486481658; duration=0sec 2024-11-24T22:14:42,150 DEBUG [RS:0;bd6592c37bb2:40087-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-24T22:14:42,150 DEBUG [RS:0;bd6592c37bb2:40087-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 63842f73863e9e7b9587486b122e6aae:A 2024-11-24T22:14:42,157 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45929 is added to blk_1073742047_1223 (size=13187) 2024-11-24T22:14:42,162 DEBUG [RS:0;bd6592c37bb2:40087-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/63842f73863e9e7b9587486b122e6aae/.tmp/C/0502b36d90f4444c9da4fb69ebf63c97 as hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/63842f73863e9e7b9587486b122e6aae/C/0502b36d90f4444c9da4fb69ebf63c97 2024-11-24T22:14:42,169 INFO [RS:0;bd6592c37bb2:40087-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 63842f73863e9e7b9587486b122e6aae/C of 63842f73863e9e7b9587486b122e6aae into 0502b36d90f4444c9da4fb69ebf63c97(size=12.9 K), total size for store is 12.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-24T22:14:42,169 DEBUG [RS:0;bd6592c37bb2:40087-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 63842f73863e9e7b9587486b122e6aae: 2024-11-24T22:14:42,169 INFO [RS:0;bd6592c37bb2:40087-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732486460633.63842f73863e9e7b9587486b122e6aae., storeName=63842f73863e9e7b9587486b122e6aae/C, priority=13, startTime=1732486481659; duration=0sec 2024-11-24T22:14:42,169 DEBUG [RS:0;bd6592c37bb2:40087-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-24T22:14:42,169 DEBUG [RS:0;bd6592c37bb2:40087-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 63842f73863e9e7b9587486b122e6aae:C 2024-11-24T22:14:42,177 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T22:14:42,182 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024112466703d3eec03428db38bbc64c672f9c9_63842f73863e9e7b9587486b122e6aae to hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024112466703d3eec03428db38bbc64c672f9c9_63842f73863e9e7b9587486b122e6aae 2024-11-24T22:14:42,184 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/63842f73863e9e7b9587486b122e6aae/.tmp/A/4e39768966df4af1ba07134b48a32c6e, store: [table=TestAcidGuarantees family=A region=63842f73863e9e7b9587486b122e6aae] 2024-11-24T22:14:42,185 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/63842f73863e9e7b9587486b122e6aae/.tmp/A/4e39768966df4af1ba07134b48a32c6e is 175, key is test_row_0/A:col10/1732486481761/Put/seqid=0 2024-11-24T22:14:42,195 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45929 is added to blk_1073742048_1224 (size=39949) 2024-11-24T22:14:42,200 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=417, memsize=49.2 K, hasBloomFilter=true, into tmp file hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/63842f73863e9e7b9587486b122e6aae/.tmp/A/4e39768966df4af1ba07134b48a32c6e 2024-11-24T22:14:42,211 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/63842f73863e9e7b9587486b122e6aae/.tmp/B/93cf1c13def944adb5dbfacdde2a8cb0 is 50, key is test_row_0/B:col10/1732486481761/Put/seqid=0 2024-11-24T22:14:42,234 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45929 is added to blk_1073742049_1225 (size=12301) 2024-11-24T22:14:42,236 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=49.20 KB at sequenceid=417 (bloomFilter=true), to=hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/63842f73863e9e7b9587486b122e6aae/.tmp/B/93cf1c13def944adb5dbfacdde2a8cb0 2024-11-24T22:14:42,251 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/63842f73863e9e7b9587486b122e6aae/.tmp/C/a8fd58c0130242109584ecd578f8cc8c is 50, key is test_row_0/C:col10/1732486481761/Put/seqid=0 2024-11-24T22:14:42,262 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45929 is added to blk_1073742050_1226 (size=12301) 2024-11-24T22:14:42,414 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40087 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=63842f73863e9e7b9587486b122e6aae, server=bd6592c37bb2,40087,1732486423713 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-24T22:14:42,414 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40087 {}] ipc.CallRunner(138): callId: 170 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40244 deadline: 1732486542411, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=63842f73863e9e7b9587486b122e6aae, server=bd6592c37bb2,40087,1732486423713 2024-11-24T22:14:42,414 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40087 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=63842f73863e9e7b9587486b122e6aae, server=bd6592c37bb2,40087,1732486423713 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-24T22:14:42,414 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732486460633.63842f73863e9e7b9587486b122e6aae., hostname=bd6592c37bb2,40087,1732486423713, seqNum=5 , the old value is region=TestAcidGuarantees,,1732486460633.63842f73863e9e7b9587486b122e6aae., hostname=bd6592c37bb2,40087,1732486423713, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=63842f73863e9e7b9587486b122e6aae, server=bd6592c37bb2,40087,1732486423713 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T22:14:42,414 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40087 {}] ipc.CallRunner(138): callId: 96 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40202 deadline: 1732486542411, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=63842f73863e9e7b9587486b122e6aae, server=bd6592c37bb2,40087,1732486423713 2024-11-24T22:14:42,414 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732486460633.63842f73863e9e7b9587486b122e6aae., hostname=bd6592c37bb2,40087,1732486423713, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=63842f73863e9e7b9587486b122e6aae, server=bd6592c37bb2,40087,1732486423713 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T22:14:42,414 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732486460633.63842f73863e9e7b9587486b122e6aae., hostname=bd6592c37bb2,40087,1732486423713, seqNum=5 because the exception is null or not the one we care about 2024-11-24T22:14:42,415 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40087 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=63842f73863e9e7b9587486b122e6aae, server=bd6592c37bb2,40087,1732486423713 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-24T22:14:42,415 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40087 {}] ipc.CallRunner(138): callId: 182 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40206 deadline: 1732486542411, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=63842f73863e9e7b9587486b122e6aae, server=bd6592c37bb2,40087,1732486423713 2024-11-24T22:14:42,415 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732486460633.63842f73863e9e7b9587486b122e6aae., hostname=bd6592c37bb2,40087,1732486423713, seqNum=5 , the old value is region=TestAcidGuarantees,,1732486460633.63842f73863e9e7b9587486b122e6aae., hostname=bd6592c37bb2,40087,1732486423713, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=63842f73863e9e7b9587486b122e6aae, server=bd6592c37bb2,40087,1732486423713 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T22:14:42,415 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732486460633.63842f73863e9e7b9587486b122e6aae., hostname=bd6592c37bb2,40087,1732486423713, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=63842f73863e9e7b9587486b122e6aae, server=bd6592c37bb2,40087,1732486423713 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T22:14:42,415 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732486460633.63842f73863e9e7b9587486b122e6aae., hostname=bd6592c37bb2,40087,1732486423713, seqNum=5 because the exception is null or not the one we care about 2024-11-24T22:14:42,416 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732486460633.63842f73863e9e7b9587486b122e6aae., hostname=bd6592c37bb2,40087,1732486423713, seqNum=5 , the old value is region=TestAcidGuarantees,,1732486460633.63842f73863e9e7b9587486b122e6aae., hostname=bd6592c37bb2,40087,1732486423713, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=63842f73863e9e7b9587486b122e6aae, server=bd6592c37bb2,40087,1732486423713 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T22:14:42,416 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732486460633.63842f73863e9e7b9587486b122e6aae., hostname=bd6592c37bb2,40087,1732486423713, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=63842f73863e9e7b9587486b122e6aae, server=bd6592c37bb2,40087,1732486423713 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T22:14:42,416 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732486460633.63842f73863e9e7b9587486b122e6aae., hostname=bd6592c37bb2,40087,1732486423713, seqNum=5 because the exception is null or not the one we care about 2024-11-24T22:14:42,663 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=49.20 KB at sequenceid=417 (bloomFilter=true), to=hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/63842f73863e9e7b9587486b122e6aae/.tmp/C/a8fd58c0130242109584ecd578f8cc8c 2024-11-24T22:14:42,671 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/63842f73863e9e7b9587486b122e6aae/.tmp/A/4e39768966df4af1ba07134b48a32c6e as hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/63842f73863e9e7b9587486b122e6aae/A/4e39768966df4af1ba07134b48a32c6e 2024-11-24T22:14:42,675 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/63842f73863e9e7b9587486b122e6aae/A/4e39768966df4af1ba07134b48a32c6e, entries=200, sequenceid=417, filesize=39.0 K 2024-11-24T22:14:42,676 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/63842f73863e9e7b9587486b122e6aae/.tmp/B/93cf1c13def944adb5dbfacdde2a8cb0 as hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/63842f73863e9e7b9587486b122e6aae/B/93cf1c13def944adb5dbfacdde2a8cb0 2024-11-24T22:14:42,683 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/63842f73863e9e7b9587486b122e6aae/B/93cf1c13def944adb5dbfacdde2a8cb0, entries=150, sequenceid=417, filesize=12.0 K 2024-11-24T22:14:42,683 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/63842f73863e9e7b9587486b122e6aae/.tmp/C/a8fd58c0130242109584ecd578f8cc8c as hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/63842f73863e9e7b9587486b122e6aae/C/a8fd58c0130242109584ecd578f8cc8c 2024-11-24T22:14:42,688 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/63842f73863e9e7b9587486b122e6aae/C/a8fd58c0130242109584ecd578f8cc8c, entries=150, sequenceid=417, filesize=12.0 K 2024-11-24T22:14:42,689 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~147.60 KB/151140, heapSize ~410.72 KB/420576, currentSize=40.25 KB/41220 for 63842f73863e9e7b9587486b122e6aae in 925ms, sequenceid=417, compaction requested=false 2024-11-24T22:14:42,689 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 63842f73863e9e7b9587486b122e6aae: 2024-11-24T22:14:42,761 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37701 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=48 2024-11-24T22:14:42,762 INFO [RPCClient-NioEventLoopGroup-4-7 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-24T22:14:42,762 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37701 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-24T22:14:42,763 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37701 {}] procedure2.ProcedureExecutor(1139): Stored pid=50, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=50, table=TestAcidGuarantees 2024-11-24T22:14:42,764 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37701 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=50 2024-11-24T22:14:42,765 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=50, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=50, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-24T22:14:42,765 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=50, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=50, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-24T22:14:42,766 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=51, ppid=50, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-24T22:14:42,871 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37701 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=50 2024-11-24T22:14:42,919 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=40087 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=51 2024-11-24T22:14:42,920 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bd6592c37bb2:0-2 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732486460633.63842f73863e9e7b9587486b122e6aae. 2024-11-24T22:14:42,920 INFO [RS_FLUSH_OPERATIONS-regionserver/bd6592c37bb2:0-2 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.HRegion(2902): Flushing 63842f73863e9e7b9587486b122e6aae 3/3 column families, dataSize=40.25 KB heapSize=120.28 KB 2024-11-24T22:14:42,920 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bd6592c37bb2:0-2 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 63842f73863e9e7b9587486b122e6aae, store=A 2024-11-24T22:14:42,920 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bd6592c37bb2:0-2 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-24T22:14:42,920 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bd6592c37bb2:0-2 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 63842f73863e9e7b9587486b122e6aae, store=B 2024-11-24T22:14:42,920 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bd6592c37bb2:0-2 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-24T22:14:42,920 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bd6592c37bb2:0-2 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 63842f73863e9e7b9587486b122e6aae, store=C 2024-11-24T22:14:42,920 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bd6592c37bb2:0-2 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-24T22:14:42,923 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40087 {}] regionserver.HRegion(8855): Flush requested on 63842f73863e9e7b9587486b122e6aae 2024-11-24T22:14:42,923 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732486460633.63842f73863e9e7b9587486b122e6aae. as already flushing 2024-11-24T22:14:42,929 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bd6592c37bb2:0-2 {event_type=RS_FLUSH_REGIONS, pid=51}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241124a88f4a2c6d31436c856417b2bf385c03_63842f73863e9e7b9587486b122e6aae is 50, key is test_row_0/A:col10/1732486481773/Put/seqid=0 2024-11-24T22:14:42,937 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45929 is added to blk_1073742051_1227 (size=12454) 2024-11-24T22:14:42,962 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/bd6592c37bb2:0-5 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1732486482948, max timestamp=1732486482958Num uniques 859; 2024-11-24T22:14:42,962 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/bd6592c37bb2:0-5 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-11-24T22:14:42,963 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/bd6592c37bb2:0-4 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1732486482948, max timestamp=1732486482958Num uniques 859; 2024-11-24T22:14:42,963 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/bd6592c37bb2:0-9 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1732486482948, max timestamp=1732486482958Num uniques 859; 2024-11-24T22:14:42,963 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/bd6592c37bb2:0-4 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-11-24T22:14:42,963 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/bd6592c37bb2:0-9 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-11-24T22:14:43,002 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40087 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=63842f73863e9e7b9587486b122e6aae, server=bd6592c37bb2,40087,1732486423713 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-24T22:14:43,002 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40087 {}] ipc.CallRunner(138): callId: 181 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40244 deadline: 1732486542997, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=63842f73863e9e7b9587486b122e6aae, server=bd6592c37bb2,40087,1732486423713 2024-11-24T22:14:43,002 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40087 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=63842f73863e9e7b9587486b122e6aae, server=bd6592c37bb2,40087,1732486423713 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-24T22:14:43,002 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732486460633.63842f73863e9e7b9587486b122e6aae., hostname=bd6592c37bb2,40087,1732486423713, seqNum=5 , the old value is region=TestAcidGuarantees,,1732486460633.63842f73863e9e7b9587486b122e6aae., hostname=bd6592c37bb2,40087,1732486423713, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=63842f73863e9e7b9587486b122e6aae, server=bd6592c37bb2,40087,1732486423713 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T22:14:43,002 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732486460633.63842f73863e9e7b9587486b122e6aae., hostname=bd6592c37bb2,40087,1732486423713, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=63842f73863e9e7b9587486b122e6aae, server=bd6592c37bb2,40087,1732486423713 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T22:14:43,003 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732486460633.63842f73863e9e7b9587486b122e6aae., hostname=bd6592c37bb2,40087,1732486423713, seqNum=5 because the exception is null or not the one we care about 2024-11-24T22:14:43,003 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40087 {}] ipc.CallRunner(138): callId: 195 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40206 deadline: 1732486543001, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=63842f73863e9e7b9587486b122e6aae, server=bd6592c37bb2,40087,1732486423713 2024-11-24T22:14:43,002 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40087 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=63842f73863e9e7b9587486b122e6aae, server=bd6592c37bb2,40087,1732486423713 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-24T22:14:43,003 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40087 {}] ipc.CallRunner(138): callId: 110 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40202 deadline: 1732486543002, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=63842f73863e9e7b9587486b122e6aae, server=bd6592c37bb2,40087,1732486423713 2024-11-24T22:14:43,003 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732486460633.63842f73863e9e7b9587486b122e6aae., hostname=bd6592c37bb2,40087,1732486423713, seqNum=5 , the old value is region=TestAcidGuarantees,,1732486460633.63842f73863e9e7b9587486b122e6aae., hostname=bd6592c37bb2,40087,1732486423713, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=63842f73863e9e7b9587486b122e6aae, server=bd6592c37bb2,40087,1732486423713 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T22:14:43,003 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732486460633.63842f73863e9e7b9587486b122e6aae., hostname=bd6592c37bb2,40087,1732486423713, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=63842f73863e9e7b9587486b122e6aae, server=bd6592c37bb2,40087,1732486423713 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T22:14:43,003 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732486460633.63842f73863e9e7b9587486b122e6aae., hostname=bd6592c37bb2,40087,1732486423713, seqNum=5 because the exception is null or not the one we care about 2024-11-24T22:14:43,004 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732486460633.63842f73863e9e7b9587486b122e6aae., hostname=bd6592c37bb2,40087,1732486423713, seqNum=5 , the old value is region=TestAcidGuarantees,,1732486460633.63842f73863e9e7b9587486b122e6aae., hostname=bd6592c37bb2,40087,1732486423713, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=63842f73863e9e7b9587486b122e6aae, server=bd6592c37bb2,40087,1732486423713 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T22:14:43,004 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732486460633.63842f73863e9e7b9587486b122e6aae., hostname=bd6592c37bb2,40087,1732486423713, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=63842f73863e9e7b9587486b122e6aae, server=bd6592c37bb2,40087,1732486423713 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T22:14:43,004 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732486460633.63842f73863e9e7b9587486b122e6aae., hostname=bd6592c37bb2,40087,1732486423713, seqNum=5 because the exception is null or not the one we care about 2024-11-24T22:14:43,005 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/bd6592c37bb2:0-6 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=218, cellSize=9978, totalHeapSize=22912, min timestamp=1732486482980, max timestamp=1732486483000Num uniques 1018; 2024-11-24T22:14:43,005 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/bd6592c37bb2:0-6 {}] regionserver.CompactionPipeline(163): Suffix data size=46618, new segment data size=9978, suffix heap size=133840,new segment heap size=22912 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1018, new segment cells count=218 2024-11-24T22:14:43,006 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/bd6592c37bb2:0-8 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=218, cellSize=9978, totalHeapSize=22912, min timestamp=1732486482980, max timestamp=1732486483000Num uniques 1018; 2024-11-24T22:14:43,006 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/bd6592c37bb2:0-8 {}] regionserver.CompactionPipeline(163): Suffix data size=46618, new segment data size=9978, suffix heap size=133840,new segment heap size=22912 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1018, new segment cells count=218 2024-11-24T22:14:43,006 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/bd6592c37bb2:0-2 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=218, cellSize=9978, totalHeapSize=22912, min timestamp=1732486482980, max timestamp=1732486483000Num uniques 1018; 2024-11-24T22:14:43,006 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/bd6592c37bb2:0-2 {}] regionserver.CompactionPipeline(163): Suffix data size=46618, new segment data size=9978, suffix heap size=133840,new segment heap size=22912 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1018, new segment cells count=218 2024-11-24T22:14:43,081 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37701 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=50 2024-11-24T22:14:43,147 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40087 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=63842f73863e9e7b9587486b122e6aae, server=bd6592c37bb2,40087,1732486423713 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-24T22:14:43,148 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40087 {}] ipc.CallRunner(138): callId: 116 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40202 deadline: 1732486543140, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=63842f73863e9e7b9587486b122e6aae, server=bd6592c37bb2,40087,1732486423713 2024-11-24T22:14:43,148 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732486460633.63842f73863e9e7b9587486b122e6aae., hostname=bd6592c37bb2,40087,1732486423713, seqNum=5 , the old value is region=TestAcidGuarantees,,1732486460633.63842f73863e9e7b9587486b122e6aae., hostname=bd6592c37bb2,40087,1732486423713, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=63842f73863e9e7b9587486b122e6aae, server=bd6592c37bb2,40087,1732486423713 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T22:14:43,148 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732486460633.63842f73863e9e7b9587486b122e6aae., hostname=bd6592c37bb2,40087,1732486423713, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=63842f73863e9e7b9587486b122e6aae, server=bd6592c37bb2,40087,1732486423713 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T22:14:43,148 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732486460633.63842f73863e9e7b9587486b122e6aae., hostname=bd6592c37bb2,40087,1732486423713, seqNum=5 because the exception is null or not the one we care about 2024-11-24T22:14:43,153 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40087 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=63842f73863e9e7b9587486b122e6aae, server=bd6592c37bb2,40087,1732486423713 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-24T22:14:43,153 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40087 {}] ipc.CallRunner(138): callId: 201 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40206 deadline: 1732486543142, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=63842f73863e9e7b9587486b122e6aae, server=bd6592c37bb2,40087,1732486423713 2024-11-24T22:14:43,153 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732486460633.63842f73863e9e7b9587486b122e6aae., hostname=bd6592c37bb2,40087,1732486423713, seqNum=5 , the old value is region=TestAcidGuarantees,,1732486460633.63842f73863e9e7b9587486b122e6aae., hostname=bd6592c37bb2,40087,1732486423713, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=63842f73863e9e7b9587486b122e6aae, server=bd6592c37bb2,40087,1732486423713 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T22:14:43,153 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40087 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=63842f73863e9e7b9587486b122e6aae, server=bd6592c37bb2,40087,1732486423713 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-24T22:14:43,153 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732486460633.63842f73863e9e7b9587486b122e6aae., hostname=bd6592c37bb2,40087,1732486423713, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=63842f73863e9e7b9587486b122e6aae, server=bd6592c37bb2,40087,1732486423713 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T22:14:43,153 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732486460633.63842f73863e9e7b9587486b122e6aae., hostname=bd6592c37bb2,40087,1732486423713, seqNum=5 because the exception is null or not the one we care about 2024-11-24T22:14:43,153 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40087 {}] ipc.CallRunner(138): callId: 188 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40244 deadline: 1732486543143, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=63842f73863e9e7b9587486b122e6aae, server=bd6592c37bb2,40087,1732486423713 2024-11-24T22:14:43,153 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732486460633.63842f73863e9e7b9587486b122e6aae., hostname=bd6592c37bb2,40087,1732486423713, seqNum=5 , the old value is region=TestAcidGuarantees,,1732486460633.63842f73863e9e7b9587486b122e6aae., hostname=bd6592c37bb2,40087,1732486423713, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=63842f73863e9e7b9587486b122e6aae, server=bd6592c37bb2,40087,1732486423713 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T22:14:43,153 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732486460633.63842f73863e9e7b9587486b122e6aae., hostname=bd6592c37bb2,40087,1732486423713, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=63842f73863e9e7b9587486b122e6aae, server=bd6592c37bb2,40087,1732486423713 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T22:14:43,153 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732486460633.63842f73863e9e7b9587486b122e6aae., hostname=bd6592c37bb2,40087,1732486423713, seqNum=5 because the exception is null or not the one we care about 2024-11-24T22:14:43,255 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40087 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=63842f73863e9e7b9587486b122e6aae, server=bd6592c37bb2,40087,1732486423713 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-24T22:14:43,255 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40087 {}] ipc.CallRunner(138): callId: 117 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40202 deadline: 1732486543251, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=63842f73863e9e7b9587486b122e6aae, server=bd6592c37bb2,40087,1732486423713 2024-11-24T22:14:43,256 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732486460633.63842f73863e9e7b9587486b122e6aae., hostname=bd6592c37bb2,40087,1732486423713, seqNum=5 , the old value is region=TestAcidGuarantees,,1732486460633.63842f73863e9e7b9587486b122e6aae., hostname=bd6592c37bb2,40087,1732486423713, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=63842f73863e9e7b9587486b122e6aae, server=bd6592c37bb2,40087,1732486423713 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T22:14:43,256 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732486460633.63842f73863e9e7b9587486b122e6aae., hostname=bd6592c37bb2,40087,1732486423713, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=63842f73863e9e7b9587486b122e6aae, server=bd6592c37bb2,40087,1732486423713 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T22:14:43,256 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732486460633.63842f73863e9e7b9587486b122e6aae., hostname=bd6592c37bb2,40087,1732486423713, seqNum=5 because the exception is null or not the one we care about 2024-11-24T22:14:43,261 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40087 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=63842f73863e9e7b9587486b122e6aae, server=bd6592c37bb2,40087,1732486423713 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-24T22:14:43,262 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40087 {}] ipc.CallRunner(138): callId: 202 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40206 deadline: 1732486543261, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=63842f73863e9e7b9587486b122e6aae, server=bd6592c37bb2,40087,1732486423713 2024-11-24T22:14:43,262 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40087 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=63842f73863e9e7b9587486b122e6aae, server=bd6592c37bb2,40087,1732486423713 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-24T22:14:43,262 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40087 {}] ipc.CallRunner(138): callId: 189 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40244 deadline: 1732486543261, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=63842f73863e9e7b9587486b122e6aae, server=bd6592c37bb2,40087,1732486423713 2024-11-24T22:14:43,262 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732486460633.63842f73863e9e7b9587486b122e6aae., hostname=bd6592c37bb2,40087,1732486423713, seqNum=5 , the old value is region=TestAcidGuarantees,,1732486460633.63842f73863e9e7b9587486b122e6aae., hostname=bd6592c37bb2,40087,1732486423713, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=63842f73863e9e7b9587486b122e6aae, server=bd6592c37bb2,40087,1732486423713 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T22:14:43,262 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732486460633.63842f73863e9e7b9587486b122e6aae., hostname=bd6592c37bb2,40087,1732486423713, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=63842f73863e9e7b9587486b122e6aae, server=bd6592c37bb2,40087,1732486423713 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T22:14:43,262 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732486460633.63842f73863e9e7b9587486b122e6aae., hostname=bd6592c37bb2,40087,1732486423713, seqNum=5 because the exception is null or not the one we care about 2024-11-24T22:14:43,262 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732486460633.63842f73863e9e7b9587486b122e6aae., hostname=bd6592c37bb2,40087,1732486423713, seqNum=5 , the old value is region=TestAcidGuarantees,,1732486460633.63842f73863e9e7b9587486b122e6aae., hostname=bd6592c37bb2,40087,1732486423713, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=63842f73863e9e7b9587486b122e6aae, server=bd6592c37bb2,40087,1732486423713 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T22:14:43,262 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732486460633.63842f73863e9e7b9587486b122e6aae., hostname=bd6592c37bb2,40087,1732486423713, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=63842f73863e9e7b9587486b122e6aae, server=bd6592c37bb2,40087,1732486423713 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T22:14:43,262 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732486460633.63842f73863e9e7b9587486b122e6aae., hostname=bd6592c37bb2,40087,1732486423713, seqNum=5 because the exception is null or not the one we care about 2024-11-24T22:14:43,337 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bd6592c37bb2:0-2 {event_type=RS_FLUSH_REGIONS, pid=51}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T22:14:43,343 INFO [RS_FLUSH_OPERATIONS-regionserver/bd6592c37bb2:0-2 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241124a88f4a2c6d31436c856417b2bf385c03_63842f73863e9e7b9587486b122e6aae to hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241124a88f4a2c6d31436c856417b2bf385c03_63842f73863e9e7b9587486b122e6aae 2024-11-24T22:14:43,349 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bd6592c37bb2:0-2 {event_type=RS_FLUSH_REGIONS, pid=51}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/63842f73863e9e7b9587486b122e6aae/.tmp/A/4fbba21c7791450da1de7eb49457b3fd, store: [table=TestAcidGuarantees family=A region=63842f73863e9e7b9587486b122e6aae] 2024-11-24T22:14:43,350 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bd6592c37bb2:0-2 {event_type=RS_FLUSH_REGIONS, pid=51}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/63842f73863e9e7b9587486b122e6aae/.tmp/A/4fbba21c7791450da1de7eb49457b3fd is 175, key is test_row_0/A:col10/1732486481773/Put/seqid=0 2024-11-24T22:14:43,358 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45929 is added to blk_1073742052_1228 (size=31255) 2024-11-24T22:14:43,359 INFO [RS_FLUSH_OPERATIONS-regionserver/bd6592c37bb2:0-2 {event_type=RS_FLUSH_REGIONS, pid=51}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=429, memsize=13.4 K, hasBloomFilter=true, into tmp file hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/63842f73863e9e7b9587486b122e6aae/.tmp/A/4fbba21c7791450da1de7eb49457b3fd 2024-11-24T22:14:43,374 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bd6592c37bb2:0-2 {event_type=RS_FLUSH_REGIONS, pid=51}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/63842f73863e9e7b9587486b122e6aae/.tmp/B/db681ed797f14c9092acfa530ecefd95 is 50, key is test_row_0/B:col10/1732486481773/Put/seqid=0 2024-11-24T22:14:43,385 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45929 is added to blk_1073742053_1229 (size=12301) 2024-11-24T22:14:43,390 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37701 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=50 2024-11-24T22:14:43,462 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40087 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=63842f73863e9e7b9587486b122e6aae, server=bd6592c37bb2,40087,1732486423713 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-24T22:14:43,463 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40087 {}] ipc.CallRunner(138): callId: 118 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40202 deadline: 1732486543461, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=63842f73863e9e7b9587486b122e6aae, server=bd6592c37bb2,40087,1732486423713 2024-11-24T22:14:43,463 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732486460633.63842f73863e9e7b9587486b122e6aae., hostname=bd6592c37bb2,40087,1732486423713, seqNum=5 , the old value is region=TestAcidGuarantees,,1732486460633.63842f73863e9e7b9587486b122e6aae., hostname=bd6592c37bb2,40087,1732486423713, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=63842f73863e9e7b9587486b122e6aae, server=bd6592c37bb2,40087,1732486423713 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T22:14:43,463 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732486460633.63842f73863e9e7b9587486b122e6aae., hostname=bd6592c37bb2,40087,1732486423713, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=63842f73863e9e7b9587486b122e6aae, server=bd6592c37bb2,40087,1732486423713 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T22:14:43,463 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732486460633.63842f73863e9e7b9587486b122e6aae., hostname=bd6592c37bb2,40087,1732486423713, seqNum=5 because the exception is null or not the one we care about 2024-11-24T22:14:43,474 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40087 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=63842f73863e9e7b9587486b122e6aae, server=bd6592c37bb2,40087,1732486423713 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-24T22:14:43,474 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40087 {}] ipc.CallRunner(138): callId: 203 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40206 deadline: 1732486543470, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=63842f73863e9e7b9587486b122e6aae, server=bd6592c37bb2,40087,1732486423713 2024-11-24T22:14:43,474 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40087 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=63842f73863e9e7b9587486b122e6aae, server=bd6592c37bb2,40087,1732486423713 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-24T22:14:43,475 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732486460633.63842f73863e9e7b9587486b122e6aae., hostname=bd6592c37bb2,40087,1732486423713, seqNum=5 , the old value is region=TestAcidGuarantees,,1732486460633.63842f73863e9e7b9587486b122e6aae., hostname=bd6592c37bb2,40087,1732486423713, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=63842f73863e9e7b9587486b122e6aae, server=bd6592c37bb2,40087,1732486423713 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T22:14:43,475 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732486460633.63842f73863e9e7b9587486b122e6aae., hostname=bd6592c37bb2,40087,1732486423713, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=63842f73863e9e7b9587486b122e6aae, server=bd6592c37bb2,40087,1732486423713 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T22:14:43,475 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40087 {}] ipc.CallRunner(138): callId: 190 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40244 deadline: 1732486543470, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=63842f73863e9e7b9587486b122e6aae, server=bd6592c37bb2,40087,1732486423713 2024-11-24T22:14:43,475 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732486460633.63842f73863e9e7b9587486b122e6aae., hostname=bd6592c37bb2,40087,1732486423713, seqNum=5 because the exception is null or not the one we care about 2024-11-24T22:14:43,475 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732486460633.63842f73863e9e7b9587486b122e6aae., hostname=bd6592c37bb2,40087,1732486423713, seqNum=5 , the old value is region=TestAcidGuarantees,,1732486460633.63842f73863e9e7b9587486b122e6aae., hostname=bd6592c37bb2,40087,1732486423713, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=63842f73863e9e7b9587486b122e6aae, server=bd6592c37bb2,40087,1732486423713 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T22:14:43,475 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732486460633.63842f73863e9e7b9587486b122e6aae., hostname=bd6592c37bb2,40087,1732486423713, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=63842f73863e9e7b9587486b122e6aae, server=bd6592c37bb2,40087,1732486423713 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T22:14:43,475 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732486460633.63842f73863e9e7b9587486b122e6aae., hostname=bd6592c37bb2,40087,1732486423713, seqNum=5 because the exception is null or not the one we care about 2024-11-24T22:14:43,774 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40087 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=63842f73863e9e7b9587486b122e6aae, server=bd6592c37bb2,40087,1732486423713 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-24T22:14:43,775 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40087 {}] ipc.CallRunner(138): callId: 119 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40202 deadline: 1732486543771, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=63842f73863e9e7b9587486b122e6aae, server=bd6592c37bb2,40087,1732486423713 2024-11-24T22:14:43,775 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732486460633.63842f73863e9e7b9587486b122e6aae., hostname=bd6592c37bb2,40087,1732486423713, seqNum=5 , the old value is region=TestAcidGuarantees,,1732486460633.63842f73863e9e7b9587486b122e6aae., hostname=bd6592c37bb2,40087,1732486423713, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=63842f73863e9e7b9587486b122e6aae, server=bd6592c37bb2,40087,1732486423713 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T22:14:43,775 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732486460633.63842f73863e9e7b9587486b122e6aae., hostname=bd6592c37bb2,40087,1732486423713, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=63842f73863e9e7b9587486b122e6aae, server=bd6592c37bb2,40087,1732486423713 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T22:14:43,775 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732486460633.63842f73863e9e7b9587486b122e6aae., hostname=bd6592c37bb2,40087,1732486423713, seqNum=5 because the exception is null or not the one we care about 2024-11-24T22:14:43,782 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40087 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=63842f73863e9e7b9587486b122e6aae, server=bd6592c37bb2,40087,1732486423713 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-24T22:14:43,782 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40087 {}] ipc.CallRunner(138): callId: 204 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40206 deadline: 1732486543781, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=63842f73863e9e7b9587486b122e6aae, server=bd6592c37bb2,40087,1732486423713 2024-11-24T22:14:43,783 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40087 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=63842f73863e9e7b9587486b122e6aae, server=bd6592c37bb2,40087,1732486423713 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-24T22:14:43,783 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732486460633.63842f73863e9e7b9587486b122e6aae., hostname=bd6592c37bb2,40087,1732486423713, seqNum=5 , the old value is region=TestAcidGuarantees,,1732486460633.63842f73863e9e7b9587486b122e6aae., hostname=bd6592c37bb2,40087,1732486423713, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=63842f73863e9e7b9587486b122e6aae, server=bd6592c37bb2,40087,1732486423713 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T22:14:43,783 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40087 {}] ipc.CallRunner(138): callId: 191 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40244 deadline: 1732486543781, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=63842f73863e9e7b9587486b122e6aae, server=bd6592c37bb2,40087,1732486423713 2024-11-24T22:14:43,783 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732486460633.63842f73863e9e7b9587486b122e6aae., hostname=bd6592c37bb2,40087,1732486423713, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=63842f73863e9e7b9587486b122e6aae, server=bd6592c37bb2,40087,1732486423713 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T22:14:43,783 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732486460633.63842f73863e9e7b9587486b122e6aae., hostname=bd6592c37bb2,40087,1732486423713, seqNum=5 because the exception is null or not the one we care about 2024-11-24T22:14:43,783 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732486460633.63842f73863e9e7b9587486b122e6aae., hostname=bd6592c37bb2,40087,1732486423713, seqNum=5 , the old value is region=TestAcidGuarantees,,1732486460633.63842f73863e9e7b9587486b122e6aae., hostname=bd6592c37bb2,40087,1732486423713, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=63842f73863e9e7b9587486b122e6aae, server=bd6592c37bb2,40087,1732486423713 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T22:14:43,783 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732486460633.63842f73863e9e7b9587486b122e6aae., hostname=bd6592c37bb2,40087,1732486423713, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=63842f73863e9e7b9587486b122e6aae, server=bd6592c37bb2,40087,1732486423713 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T22:14:43,783 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732486460633.63842f73863e9e7b9587486b122e6aae., hostname=bd6592c37bb2,40087,1732486423713, seqNum=5 because the exception is null or not the one we care about 2024-11-24T22:14:43,787 INFO [RS_FLUSH_OPERATIONS-regionserver/bd6592c37bb2:0-2 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=13.42 KB at sequenceid=429 (bloomFilter=true), to=hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/63842f73863e9e7b9587486b122e6aae/.tmp/B/db681ed797f14c9092acfa530ecefd95 2024-11-24T22:14:43,798 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bd6592c37bb2:0-2 {event_type=RS_FLUSH_REGIONS, pid=51}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/63842f73863e9e7b9587486b122e6aae/.tmp/C/46ae7d5ce9c54d38a636ba8fde779f36 is 50, key is test_row_0/C:col10/1732486481773/Put/seqid=0 2024-11-24T22:14:43,805 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45929 is added to blk_1073742054_1230 (size=12301) 2024-11-24T22:14:43,860 INFO [Thread-596 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-596. 2024-11-24T22:14:43,860 DEBUG [Thread-596 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicGetReader.workDone(AcidGuaranteesTestTool.java:234) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-24T22:14:43,860 DEBUG [Thread-596 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-24T22:14:43,860 DEBUG [Thread-596 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-24T22:14:43,860 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-24T22:14:43,864 INFO [Thread-598 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-598. 2024-11-24T22:14:43,864 DEBUG [Thread-598 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicGetReader.workDone(AcidGuaranteesTestTool.java:234) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-24T22:14:43,864 DEBUG [Thread-598 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-24T22:14:43,864 DEBUG [Thread-598 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-24T22:14:43,864 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-24T22:14:43,871 INFO [Thread-602 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-602. 2024-11-24T22:14:43,871 DEBUG [Thread-602 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicScanReader.workDone(AcidGuaranteesTestTool.java:302) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-24T22:14:43,871 DEBUG [Thread-602 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-24T22:14:43,871 DEBUG [Thread-602 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-24T22:14:43,871 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-24T22:14:43,879 INFO [Thread-600 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-600. 2024-11-24T22:14:43,879 DEBUG [Thread-600 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicScanReader.workDone(AcidGuaranteesTestTool.java:302) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-24T22:14:43,879 DEBUG [Thread-600 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-24T22:14:43,879 DEBUG [Thread-600 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-24T22:14:43,879 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-24T22:14:43,900 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37701 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=50 2024-11-24T22:14:44,205 INFO [RS_FLUSH_OPERATIONS-regionserver/bd6592c37bb2:0-2 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=13.42 KB at sequenceid=429 (bloomFilter=true), to=hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/63842f73863e9e7b9587486b122e6aae/.tmp/C/46ae7d5ce9c54d38a636ba8fde779f36 2024-11-24T22:14:44,210 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bd6592c37bb2:0-2 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/63842f73863e9e7b9587486b122e6aae/.tmp/A/4fbba21c7791450da1de7eb49457b3fd as hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/63842f73863e9e7b9587486b122e6aae/A/4fbba21c7791450da1de7eb49457b3fd 2024-11-24T22:14:44,215 INFO [RS_FLUSH_OPERATIONS-regionserver/bd6592c37bb2:0-2 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/63842f73863e9e7b9587486b122e6aae/A/4fbba21c7791450da1de7eb49457b3fd, entries=150, sequenceid=429, filesize=30.5 K 2024-11-24T22:14:44,216 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bd6592c37bb2:0-2 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/63842f73863e9e7b9587486b122e6aae/.tmp/B/db681ed797f14c9092acfa530ecefd95 as hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/63842f73863e9e7b9587486b122e6aae/B/db681ed797f14c9092acfa530ecefd95 2024-11-24T22:14:44,224 INFO [RS_FLUSH_OPERATIONS-regionserver/bd6592c37bb2:0-2 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/63842f73863e9e7b9587486b122e6aae/B/db681ed797f14c9092acfa530ecefd95, entries=150, sequenceid=429, filesize=12.0 K 2024-11-24T22:14:44,224 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bd6592c37bb2:0-2 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/63842f73863e9e7b9587486b122e6aae/.tmp/C/46ae7d5ce9c54d38a636ba8fde779f36 as hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/63842f73863e9e7b9587486b122e6aae/C/46ae7d5ce9c54d38a636ba8fde779f36 2024-11-24T22:14:44,229 INFO [RS_FLUSH_OPERATIONS-regionserver/bd6592c37bb2:0-2 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/63842f73863e9e7b9587486b122e6aae/C/46ae7d5ce9c54d38a636ba8fde779f36, entries=150, sequenceid=429, filesize=12.0 K 2024-11-24T22:14:44,230 INFO [RS_FLUSH_OPERATIONS-regionserver/bd6592c37bb2:0-2 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.HRegion(3140): Finished flush of dataSize ~40.25 KB/41220, heapSize ~120.23 KB/123120, currentSize=140.89 KB/144270 for 63842f73863e9e7b9587486b122e6aae in 1310ms, sequenceid=429, compaction requested=true 2024-11-24T22:14:44,230 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bd6592c37bb2:0-2 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.HRegion(2603): Flush status journal for 63842f73863e9e7b9587486b122e6aae: 2024-11-24T22:14:44,230 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bd6592c37bb2:0-2 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732486460633.63842f73863e9e7b9587486b122e6aae. 2024-11-24T22:14:44,230 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bd6592c37bb2:0-2 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=51 2024-11-24T22:14:44,230 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37701 {}] master.HMaster(4169): Remote procedure done, pid=51 2024-11-24T22:14:44,233 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=51, resume processing ppid=50 2024-11-24T22:14:44,233 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=51, ppid=50, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.4650 sec 2024-11-24T22:14:44,235 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=50, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=50, table=TestAcidGuarantees in 1.4720 sec 2024-11-24T22:14:44,283 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40087 {}] regionserver.HRegion(8855): Flush requested on 63842f73863e9e7b9587486b122e6aae 2024-11-24T22:14:44,283 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 63842f73863e9e7b9587486b122e6aae 3/3 column families, dataSize=147.60 KB heapSize=420 KB 2024-11-24T22:14:44,284 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 63842f73863e9e7b9587486b122e6aae, store=A 2024-11-24T22:14:44,284 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=3, new segment=null 2024-11-24T22:14:44,284 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 63842f73863e9e7b9587486b122e6aae, store=B 2024-11-24T22:14:44,284 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=3, new segment=null 2024-11-24T22:14:44,284 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 63842f73863e9e7b9587486b122e6aae, store=C 2024-11-24T22:14:44,284 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=3, new segment=null 2024-11-24T22:14:44,287 INFO [Thread-589 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-589. 2024-11-24T22:14:44,287 DEBUG [Thread-589 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-24T22:14:44,287 DEBUG [Thread-589 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-24T22:14:44,287 DEBUG [Thread-589 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-24T22:14:44,287 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-24T22:14:44,293 INFO [Thread-587 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-587. 2024-11-24T22:14:44,293 DEBUG [Thread-587 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-24T22:14:44,293 DEBUG [Thread-587 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-24T22:14:44,293 DEBUG [Thread-587 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-24T22:14:44,293 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-24T22:14:44,294 INFO [Thread-585 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-585. 2024-11-24T22:14:44,294 DEBUG [Thread-585 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-24T22:14:44,295 DEBUG [Thread-585 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-24T22:14:44,295 DEBUG [Thread-585 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-24T22:14:44,295 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-24T22:14:44,297 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411240dd7075793d44845b489eb8dcfe3af1c_63842f73863e9e7b9587486b122e6aae is 50, key is test_row_0/A:col10/1732486484282/Put/seqid=0 2024-11-24T22:14:44,301 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45929 is added to blk_1073742055_1231 (size=12454) 2024-11-24T22:14:44,702 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T22:14:44,706 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411240dd7075793d44845b489eb8dcfe3af1c_63842f73863e9e7b9587486b122e6aae to hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411240dd7075793d44845b489eb8dcfe3af1c_63842f73863e9e7b9587486b122e6aae 2024-11-24T22:14:44,707 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/63842f73863e9e7b9587486b122e6aae/.tmp/A/80e746b9efc84951a0532b114e66180a, store: [table=TestAcidGuarantees family=A region=63842f73863e9e7b9587486b122e6aae] 2024-11-24T22:14:44,707 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/63842f73863e9e7b9587486b122e6aae/.tmp/A/80e746b9efc84951a0532b114e66180a is 175, key is test_row_0/A:col10/1732486484282/Put/seqid=0 2024-11-24T22:14:44,712 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45929 is added to blk_1073742056_1232 (size=31255) 2024-11-24T22:14:44,910 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37701 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=50 2024-11-24T22:14:44,911 INFO [RPCClient-NioEventLoopGroup-4-7 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-24T22:14:45,113 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=484, memsize=49.2 K, hasBloomFilter=true, into tmp file hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/63842f73863e9e7b9587486b122e6aae/.tmp/A/80e746b9efc84951a0532b114e66180a 2024-11-24T22:14:45,120 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/63842f73863e9e7b9587486b122e6aae/.tmp/B/7c235b3244fb42a080db1fba055d1237 is 50, key is test_row_0/B:col10/1732486484282/Put/seqid=0 2024-11-24T22:14:45,124 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45929 is added to blk_1073742057_1233 (size=12301) 2024-11-24T22:14:45,525 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=49.20 KB at sequenceid=484 (bloomFilter=true), to=hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/63842f73863e9e7b9587486b122e6aae/.tmp/B/7c235b3244fb42a080db1fba055d1237 2024-11-24T22:14:45,532 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/63842f73863e9e7b9587486b122e6aae/.tmp/C/9432eb988e004ad091e33fa3661cc3f0 is 50, key is test_row_0/C:col10/1732486484282/Put/seqid=0 2024-11-24T22:14:45,536 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45929 is added to blk_1073742058_1234 (size=12301) 2024-11-24T22:14:45,937 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=49.20 KB at sequenceid=484 (bloomFilter=true), to=hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/63842f73863e9e7b9587486b122e6aae/.tmp/C/9432eb988e004ad091e33fa3661cc3f0 2024-11-24T22:14:45,941 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/63842f73863e9e7b9587486b122e6aae/.tmp/A/80e746b9efc84951a0532b114e66180a as hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/63842f73863e9e7b9587486b122e6aae/A/80e746b9efc84951a0532b114e66180a 2024-11-24T22:14:45,945 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/63842f73863e9e7b9587486b122e6aae/A/80e746b9efc84951a0532b114e66180a, entries=150, sequenceid=484, filesize=30.5 K 2024-11-24T22:14:45,946 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/63842f73863e9e7b9587486b122e6aae/.tmp/B/7c235b3244fb42a080db1fba055d1237 as hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/63842f73863e9e7b9587486b122e6aae/B/7c235b3244fb42a080db1fba055d1237 2024-11-24T22:14:45,950 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/63842f73863e9e7b9587486b122e6aae/B/7c235b3244fb42a080db1fba055d1237, entries=150, sequenceid=484, filesize=12.0 K 2024-11-24T22:14:45,951 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/63842f73863e9e7b9587486b122e6aae/.tmp/C/9432eb988e004ad091e33fa3661cc3f0 as hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/63842f73863e9e7b9587486b122e6aae/C/9432eb988e004ad091e33fa3661cc3f0 2024-11-24T22:14:45,954 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/63842f73863e9e7b9587486b122e6aae/C/9432eb988e004ad091e33fa3661cc3f0, entries=150, sequenceid=484, filesize=12.0 K 2024-11-24T22:14:45,955 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~147.60 KB/151140, heapSize ~419.95 KB/430032, currentSize=13.42 KB/13740 for 63842f73863e9e7b9587486b122e6aae in 1672ms, sequenceid=484, compaction requested=true 2024-11-24T22:14:45,955 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 63842f73863e9e7b9587486b122e6aae: 2024-11-24T22:14:45,955 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 63842f73863e9e7b9587486b122e6aae:A, priority=-2147483648, current under compaction store size is 1 2024-11-24T22:14:45,955 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-24T22:14:45,955 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 63842f73863e9e7b9587486b122e6aae:B, priority=-2147483648, current under compaction store size is 2 2024-11-24T22:14:45,955 DEBUG [RS:0;bd6592c37bb2:40087-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-24T22:14:45,955 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-24T22:14:45,955 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 63842f73863e9e7b9587486b122e6aae:C, priority=-2147483648, current under compaction store size is 3 2024-11-24T22:14:45,955 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-24T22:14:45,955 DEBUG [RS:0;bd6592c37bb2:40087-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-24T22:14:45,956 DEBUG [RS:0;bd6592c37bb2:40087-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 134600 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-24T22:14:45,957 DEBUG [RS:0;bd6592c37bb2:40087-shortCompactions-0 {}] regionserver.HStore(1541): 63842f73863e9e7b9587486b122e6aae/A is initiating minor compaction (all files) 2024-11-24T22:14:45,957 INFO [RS:0;bd6592c37bb2:40087-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 63842f73863e9e7b9587486b122e6aae/A in TestAcidGuarantees,,1732486460633.63842f73863e9e7b9587486b122e6aae. 2024-11-24T22:14:45,957 INFO [RS:0;bd6592c37bb2:40087-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/63842f73863e9e7b9587486b122e6aae/A/2b8c21ff057c4163b79d571652f1f38c, hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/63842f73863e9e7b9587486b122e6aae/A/4e39768966df4af1ba07134b48a32c6e, hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/63842f73863e9e7b9587486b122e6aae/A/4fbba21c7791450da1de7eb49457b3fd, hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/63842f73863e9e7b9587486b122e6aae/A/80e746b9efc84951a0532b114e66180a] into tmpdir=hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/63842f73863e9e7b9587486b122e6aae/.tmp, totalSize=131.4 K 2024-11-24T22:14:45,957 INFO [RS:0;bd6592c37bb2:40087-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=12 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1732486460633.63842f73863e9e7b9587486b122e6aae. 2024-11-24T22:14:45,957 DEBUG [RS:0;bd6592c37bb2:40087-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1732486460633.63842f73863e9e7b9587486b122e6aae. files: [hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/63842f73863e9e7b9587486b122e6aae/A/2b8c21ff057c4163b79d571652f1f38c, hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/63842f73863e9e7b9587486b122e6aae/A/4e39768966df4af1ba07134b48a32c6e, hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/63842f73863e9e7b9587486b122e6aae/A/4fbba21c7791450da1de7eb49457b3fd, hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/63842f73863e9e7b9587486b122e6aae/A/80e746b9efc84951a0532b114e66180a] 2024-11-24T22:14:45,957 DEBUG [RS:0;bd6592c37bb2:40087-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 50090 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-24T22:14:45,957 DEBUG [RS:0;bd6592c37bb2:40087-shortCompactions-0 {}] compactions.Compactor(225): Compacting 2b8c21ff057c4163b79d571652f1f38c, keycount=150, bloomtype=ROW, size=31.4 K, encoding=NONE, compression=NONE, seqNum=381, earliestPutTs=1732486481512 2024-11-24T22:14:45,957 DEBUG [RS:0;bd6592c37bb2:40087-longCompactions-0 {}] regionserver.HStore(1541): 63842f73863e9e7b9587486b122e6aae/B is initiating minor compaction (all files) 2024-11-24T22:14:45,957 INFO [RS:0;bd6592c37bb2:40087-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 63842f73863e9e7b9587486b122e6aae/B in TestAcidGuarantees,,1732486460633.63842f73863e9e7b9587486b122e6aae. 2024-11-24T22:14:45,957 INFO [RS:0;bd6592c37bb2:40087-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/63842f73863e9e7b9587486b122e6aae/B/0be02238389c498580e6164758290ed8, hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/63842f73863e9e7b9587486b122e6aae/B/93cf1c13def944adb5dbfacdde2a8cb0, hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/63842f73863e9e7b9587486b122e6aae/B/db681ed797f14c9092acfa530ecefd95, hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/63842f73863e9e7b9587486b122e6aae/B/7c235b3244fb42a080db1fba055d1237] into tmpdir=hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/63842f73863e9e7b9587486b122e6aae/.tmp, totalSize=48.9 K 2024-11-24T22:14:45,957 DEBUG [RS:0;bd6592c37bb2:40087-shortCompactions-0 {}] compactions.Compactor(225): Compacting 4e39768966df4af1ba07134b48a32c6e, keycount=200, bloomtype=ROW, size=39.0 K, encoding=NONE, compression=NONE, seqNum=417, earliestPutTs=1732486481646 2024-11-24T22:14:45,958 DEBUG [RS:0;bd6592c37bb2:40087-longCompactions-0 {}] compactions.Compactor(225): Compacting 0be02238389c498580e6164758290ed8, keycount=150, bloomtype=ROW, size=12.9 K, encoding=NONE, compression=NONE, seqNum=381, earliestPutTs=1732486481512 2024-11-24T22:14:45,958 DEBUG [RS:0;bd6592c37bb2:40087-shortCompactions-0 {}] compactions.Compactor(225): Compacting 4fbba21c7791450da1de7eb49457b3fd, keycount=150, bloomtype=ROW, size=30.5 K, encoding=NONE, compression=NONE, seqNum=429, earliestPutTs=1732486481773 2024-11-24T22:14:45,958 DEBUG [RS:0;bd6592c37bb2:40087-longCompactions-0 {}] compactions.Compactor(225): Compacting 93cf1c13def944adb5dbfacdde2a8cb0, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=417, earliestPutTs=1732486481646 2024-11-24T22:14:45,958 DEBUG [RS:0;bd6592c37bb2:40087-shortCompactions-0 {}] compactions.Compactor(225): Compacting 80e746b9efc84951a0532b114e66180a, keycount=150, bloomtype=ROW, size=30.5 K, encoding=NONE, compression=NONE, seqNum=484, earliestPutTs=1732486483132 2024-11-24T22:14:45,958 DEBUG [RS:0;bd6592c37bb2:40087-longCompactions-0 {}] compactions.Compactor(225): Compacting db681ed797f14c9092acfa530ecefd95, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=429, earliestPutTs=1732486481773 2024-11-24T22:14:45,959 DEBUG [RS:0;bd6592c37bb2:40087-longCompactions-0 {}] compactions.Compactor(225): Compacting 7c235b3244fb42a080db1fba055d1237, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=484, earliestPutTs=1732486483132 2024-11-24T22:14:45,968 INFO [RS:0;bd6592c37bb2:40087-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 63842f73863e9e7b9587486b122e6aae#B#compaction#189 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-24T22:14:45,969 DEBUG [RS:0;bd6592c37bb2:40087-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/63842f73863e9e7b9587486b122e6aae/.tmp/B/e6b46686d51a490199dd02551a78110d is 50, key is test_row_0/B:col10/1732486484282/Put/seqid=0 2024-11-24T22:14:45,972 INFO [RS:0;bd6592c37bb2:40087-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=63842f73863e9e7b9587486b122e6aae] 2024-11-24T22:14:45,973 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45929 is added to blk_1073742059_1235 (size=13323) 2024-11-24T22:14:45,974 DEBUG [RS:0;bd6592c37bb2:40087-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e2024112452eed9ffb62c44e38212e8425a0fdf11_63842f73863e9e7b9587486b122e6aae store=[table=TestAcidGuarantees family=A region=63842f73863e9e7b9587486b122e6aae] 2024-11-24T22:14:45,999 DEBUG [RS:0;bd6592c37bb2:40087-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e2024112452eed9ffb62c44e38212e8425a0fdf11_63842f73863e9e7b9587486b122e6aae, store=[table=TestAcidGuarantees family=A region=63842f73863e9e7b9587486b122e6aae] 2024-11-24T22:14:45,999 DEBUG [RS:0;bd6592c37bb2:40087-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024112452eed9ffb62c44e38212e8425a0fdf11_63842f73863e9e7b9587486b122e6aae because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=63842f73863e9e7b9587486b122e6aae] 2024-11-24T22:14:46,013 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45929 is added to blk_1073742060_1236 (size=4469) 2024-11-24T22:14:46,343 INFO [Thread-591 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-591. 2024-11-24T22:14:46,343 DEBUG [Thread-591 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-24T22:14:46,343 DEBUG [Thread-591 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-24T22:14:46,343 DEBUG [Thread-591 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-24T22:14:46,343 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-24T22:14:46,378 DEBUG [RS:0;bd6592c37bb2:40087-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/63842f73863e9e7b9587486b122e6aae/.tmp/B/e6b46686d51a490199dd02551a78110d as hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/63842f73863e9e7b9587486b122e6aae/B/e6b46686d51a490199dd02551a78110d 2024-11-24T22:14:46,383 INFO [Thread-593 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-593. 2024-11-24T22:14:46,383 DEBUG [Thread-593 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-24T22:14:46,383 DEBUG [Thread-593 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-24T22:14:46,383 DEBUG [Thread-593 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-24T22:14:46,383 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-24T22:14:46,383 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(392): Finished test. Writers: 2024-11-24T22:14:46,383 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 131 2024-11-24T22:14:46,383 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 144 2024-11-24T22:14:46,383 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 81 2024-11-24T22:14:46,383 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 19 2024-11-24T22:14:46,383 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 19 2024-11-24T22:14:46,383 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(396): Readers: 2024-11-24T22:14:46,383 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(398): read 2183 2024-11-24T22:14:46,383 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(398): read 2068 2024-11-24T22:14:46,383 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(400): Scanners: 2024-11-24T22:14:46,383 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(402): scanned 940 2024-11-24T22:14:46,383 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(403): verified 2820 rows 2024-11-24T22:14:46,383 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(402): scanned 951 2024-11-24T22:14:46,383 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(403): verified 2842 rows 2024-11-24T22:14:46,384 INFO [Time-limited test {}] client.AsyncConnectionImpl(233): Connection has been closed by Time-limited test. 2024-11-24T22:14:46,384 INFO [RS:0;bd6592c37bb2:40087-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 63842f73863e9e7b9587486b122e6aae/B of 63842f73863e9e7b9587486b122e6aae into e6b46686d51a490199dd02551a78110d(size=13.0 K), total size for store is 13.0 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-24T22:14:46,384 DEBUG [RS:0;bd6592c37bb2:40087-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 63842f73863e9e7b9587486b122e6aae: 2024-11-24T22:14:46,384 DEBUG [Time-limited test {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool.doWork(AcidGuaranteesTestTool.java:131) at org.apache.hadoop.hbase.util.AbstractHBaseTool.run(AbstractHBaseTool.java:150) at org.apache.hadoop.hbase.AcidGuaranteesTestBase.runTestAtomicity(AcidGuaranteesTestBase.java:102) at org.apache.hadoop.hbase.AcidGuaranteesTestBase.testMobMixedAtomicity(AcidGuaranteesTestBase.java:132) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) at java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) at java.base/java.lang.reflect.Method.invoke(Method.java:568) at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) at java.base/java.util.concurrent.FutureTask.run(FutureTask.java:264) at java.base/java.lang.Thread.run(Thread.java:840) 2024-11-24T22:14:46,384 INFO [RS:0;bd6592c37bb2:40087-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732486460633.63842f73863e9e7b9587486b122e6aae., storeName=63842f73863e9e7b9587486b122e6aae/B, priority=12, startTime=1732486485955; duration=0sec 2024-11-24T22:14:46,384 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-24T22:14:46,384 DEBUG [RS:0;bd6592c37bb2:40087-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-24T22:14:46,384 DEBUG [RS:0;bd6592c37bb2:40087-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 63842f73863e9e7b9587486b122e6aae:B 2024-11-24T22:14:46,384 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-24T22:14:46,384 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-24T22:14:46,384 DEBUG [RS:0;bd6592c37bb2:40087-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-24T22:14:46,385 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37701 {}] master.HMaster$13(2820): Client=jenkins//172.17.0.2 disable TestAcidGuarantees 2024-11-24T22:14:46,385 DEBUG [RS:0;bd6592c37bb2:40087-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 50090 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-24T22:14:46,386 DEBUG [RS:0;bd6592c37bb2:40087-longCompactions-0 {}] regionserver.HStore(1541): 63842f73863e9e7b9587486b122e6aae/C is initiating minor compaction (all files) 2024-11-24T22:14:46,386 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37701 {}] procedure2.ProcedureExecutor(1139): Stored pid=52, state=RUNNABLE:DISABLE_TABLE_PREPARE, hasLock=false; DisableTableProcedure table=TestAcidGuarantees 2024-11-24T22:14:46,386 INFO [RS:0;bd6592c37bb2:40087-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 63842f73863e9e7b9587486b122e6aae/C in TestAcidGuarantees,,1732486460633.63842f73863e9e7b9587486b122e6aae. 2024-11-24T22:14:46,386 INFO [RS:0;bd6592c37bb2:40087-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/63842f73863e9e7b9587486b122e6aae/C/0502b36d90f4444c9da4fb69ebf63c97, hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/63842f73863e9e7b9587486b122e6aae/C/a8fd58c0130242109584ecd578f8cc8c, hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/63842f73863e9e7b9587486b122e6aae/C/46ae7d5ce9c54d38a636ba8fde779f36, hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/63842f73863e9e7b9587486b122e6aae/C/9432eb988e004ad091e33fa3661cc3f0] into tmpdir=hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/63842f73863e9e7b9587486b122e6aae/.tmp, totalSize=48.9 K 2024-11-24T22:14:46,386 DEBUG [RS:0;bd6592c37bb2:40087-longCompactions-0 {}] compactions.Compactor(225): Compacting 0502b36d90f4444c9da4fb69ebf63c97, keycount=150, bloomtype=ROW, size=12.9 K, encoding=NONE, compression=NONE, seqNum=381, earliestPutTs=1732486481512 2024-11-24T22:14:46,387 DEBUG [RS:0;bd6592c37bb2:40087-longCompactions-0 {}] compactions.Compactor(225): Compacting a8fd58c0130242109584ecd578f8cc8c, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=417, earliestPutTs=1732486481646 2024-11-24T22:14:46,387 DEBUG [RS:0;bd6592c37bb2:40087-longCompactions-0 {}] compactions.Compactor(225): Compacting 46ae7d5ce9c54d38a636ba8fde779f36, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=429, earliestPutTs=1732486481773 2024-11-24T22:14:46,387 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37701 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=52 2024-11-24T22:14:46,388 DEBUG [RS:0;bd6592c37bb2:40087-longCompactions-0 {}] compactions.Compactor(225): Compacting 9432eb988e004ad091e33fa3661cc3f0, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=484, earliestPutTs=1732486483132 2024-11-24T22:14:46,388 DEBUG [PEWorker-5 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1732486486388"}]},"ts":"1732486486388"} 2024-11-24T22:14:46,390 INFO [PEWorker-5 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=DISABLING in hbase:meta 2024-11-24T22:14:46,390 INFO [PEWorker-5 {}] procedure.DisableTableProcedure(284): Set TestAcidGuarantees to state=DISABLING 2024-11-24T22:14:46,391 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=53, ppid=52, state=RUNNABLE:CLOSE_TABLE_REGIONS_SCHEDULE, hasLock=false; CloseTableRegionsProcedure table=TestAcidGuarantees}] 2024-11-24T22:14:46,392 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=54, ppid=53, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=63842f73863e9e7b9587486b122e6aae, UNASSIGN}] 2024-11-24T22:14:46,393 INFO [PEWorker-2 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=54, ppid=53, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=63842f73863e9e7b9587486b122e6aae, UNASSIGN 2024-11-24T22:14:46,394 INFO [PEWorker-2 {}] assignment.RegionStateStore(223): pid=54 updating hbase:meta row=63842f73863e9e7b9587486b122e6aae, regionState=CLOSING, regionLocation=bd6592c37bb2,40087,1732486423713 2024-11-24T22:14:46,395 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=54, ppid=53, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=63842f73863e9e7b9587486b122e6aae, UNASSIGN because future has completed 2024-11-24T22:14:46,396 DEBUG [PEWorker-3 {}] assignment.TransitRegionStateProcedure(375): Close region: isSplit: false: evictOnSplit: true: evictOnClose: false 2024-11-24T22:14:46,396 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=55, ppid=54, state=RUNNABLE, hasLock=false; CloseRegionProcedure 63842f73863e9e7b9587486b122e6aae, server=bd6592c37bb2,40087,1732486423713}] 2024-11-24T22:14:46,398 INFO [RS:0;bd6592c37bb2:40087-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 63842f73863e9e7b9587486b122e6aae#C#compaction#191 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-24T22:14:46,399 DEBUG [RS:0;bd6592c37bb2:40087-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/63842f73863e9e7b9587486b122e6aae/.tmp/C/6db7eb2eee4a4a6b860ce55d7660e23d is 50, key is test_row_0/C:col10/1732486484282/Put/seqid=0 2024-11-24T22:14:46,406 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45929 is added to blk_1073742061_1237 (size=13323) 2024-11-24T22:14:46,414 INFO [RS:0;bd6592c37bb2:40087-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 63842f73863e9e7b9587486b122e6aae#A#compaction#190 average throughput is 0.06 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-24T22:14:46,415 DEBUG [RS:0;bd6592c37bb2:40087-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/63842f73863e9e7b9587486b122e6aae/.tmp/A/787c43c747aa44eea1b31a88c2ef6646 is 175, key is test_row_0/A:col10/1732486484282/Put/seqid=0 2024-11-24T22:14:46,419 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45929 is added to blk_1073742062_1238 (size=32277) 2024-11-24T22:14:46,491 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37701 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=52 2024-11-24T22:14:46,548 INFO [RS_CLOSE_REGION-regionserver/bd6592c37bb2:0-0 {event_type=M_RS_CLOSE_REGION, pid=55}] handler.UnassignRegionHandler(122): Close 63842f73863e9e7b9587486b122e6aae 2024-11-24T22:14:46,548 DEBUG [RS_CLOSE_REGION-regionserver/bd6592c37bb2:0-0 {event_type=M_RS_CLOSE_REGION, pid=55}] handler.UnassignRegionHandler(136): Unassign region: split region: false: evictCache: false 2024-11-24T22:14:46,548 DEBUG [RS_CLOSE_REGION-regionserver/bd6592c37bb2:0-0 {event_type=M_RS_CLOSE_REGION, pid=55}] regionserver.HRegion(1722): Closing 63842f73863e9e7b9587486b122e6aae, disabling compactions & flushes 2024-11-24T22:14:46,548 DEBUG [RS_CLOSE_REGION-regionserver/bd6592c37bb2:0-0 {event_type=M_RS_CLOSE_REGION, pid=55}] regionserver.HRegion(1993): waiting for 2 compactions to complete for region TestAcidGuarantees,,1732486460633.63842f73863e9e7b9587486b122e6aae. 2024-11-24T22:14:46,701 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37701 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=52 2024-11-24T22:14:46,812 DEBUG [RS:0;bd6592c37bb2:40087-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/63842f73863e9e7b9587486b122e6aae/.tmp/C/6db7eb2eee4a4a6b860ce55d7660e23d as hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/63842f73863e9e7b9587486b122e6aae/C/6db7eb2eee4a4a6b860ce55d7660e23d 2024-11-24T22:14:46,818 INFO [RS:0;bd6592c37bb2:40087-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 63842f73863e9e7b9587486b122e6aae/C of 63842f73863e9e7b9587486b122e6aae into 6db7eb2eee4a4a6b860ce55d7660e23d(size=13.0 K), total size for store is 13.0 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-24T22:14:46,818 DEBUG [RS:0;bd6592c37bb2:40087-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 63842f73863e9e7b9587486b122e6aae: 2024-11-24T22:14:46,818 INFO [RS:0;bd6592c37bb2:40087-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732486460633.63842f73863e9e7b9587486b122e6aae., storeName=63842f73863e9e7b9587486b122e6aae/C, priority=12, startTime=1732486485955; duration=0sec 2024-11-24T22:14:46,818 DEBUG [RS:0;bd6592c37bb2:40087-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-24T22:14:46,818 DEBUG [RS:0;bd6592c37bb2:40087-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 63842f73863e9e7b9587486b122e6aae:C 2024-11-24T22:14:46,825 DEBUG [RS:0;bd6592c37bb2:40087-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/63842f73863e9e7b9587486b122e6aae/.tmp/A/787c43c747aa44eea1b31a88c2ef6646 as hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/63842f73863e9e7b9587486b122e6aae/A/787c43c747aa44eea1b31a88c2ef6646 2024-11-24T22:14:46,831 INFO [RS:0;bd6592c37bb2:40087-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 63842f73863e9e7b9587486b122e6aae/A of 63842f73863e9e7b9587486b122e6aae into 787c43c747aa44eea1b31a88c2ef6646(size=31.5 K), total size for store is 31.5 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-24T22:14:46,831 DEBUG [RS:0;bd6592c37bb2:40087-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 63842f73863e9e7b9587486b122e6aae: 2024-11-24T22:14:46,831 INFO [RS_CLOSE_REGION-regionserver/bd6592c37bb2:0-0 {event_type=M_RS_CLOSE_REGION, pid=55}] regionserver.HRegion(1755): Closing region TestAcidGuarantees,,1732486460633.63842f73863e9e7b9587486b122e6aae. 2024-11-24T22:14:46,831 INFO [RS:0;bd6592c37bb2:40087-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732486460633.63842f73863e9e7b9587486b122e6aae., storeName=63842f73863e9e7b9587486b122e6aae/A, priority=12, startTime=1732486485955; duration=0sec 2024-11-24T22:14:46,831 DEBUG [RS_CLOSE_REGION-regionserver/bd6592c37bb2:0-0 {event_type=M_RS_CLOSE_REGION, pid=55}] regionserver.HRegion(1776): Time limited wait for close lock on TestAcidGuarantees,,1732486460633.63842f73863e9e7b9587486b122e6aae. 2024-11-24T22:14:46,831 DEBUG [RS:0;bd6592c37bb2:40087-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-24T22:14:46,831 DEBUG [RS_CLOSE_REGION-regionserver/bd6592c37bb2:0-0 {event_type=M_RS_CLOSE_REGION, pid=55}] regionserver.HRegion(1843): Acquired close lock on TestAcidGuarantees,,1732486460633.63842f73863e9e7b9587486b122e6aae. after waiting 0 ms 2024-11-24T22:14:46,831 DEBUG [RS:0;bd6592c37bb2:40087-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 63842f73863e9e7b9587486b122e6aae:A 2024-11-24T22:14:46,831 DEBUG [RS_CLOSE_REGION-regionserver/bd6592c37bb2:0-0 {event_type=M_RS_CLOSE_REGION, pid=55}] regionserver.HRegion(1853): Updates disabled for region TestAcidGuarantees,,1732486460633.63842f73863e9e7b9587486b122e6aae. 2024-11-24T22:14:46,831 INFO [RS_CLOSE_REGION-regionserver/bd6592c37bb2:0-0 {event_type=M_RS_CLOSE_REGION, pid=55}] regionserver.HRegion(2902): Flushing 63842f73863e9e7b9587486b122e6aae 3/3 column families, dataSize=26.84 KB heapSize=80.44 KB 2024-11-24T22:14:46,832 DEBUG [RS_CLOSE_REGION-regionserver/bd6592c37bb2:0-0 {event_type=M_RS_CLOSE_REGION, pid=55}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 63842f73863e9e7b9587486b122e6aae, store=A 2024-11-24T22:14:46,832 DEBUG [RS_CLOSE_REGION-regionserver/bd6592c37bb2:0-0 {event_type=M_RS_CLOSE_REGION, pid=55}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-24T22:14:46,832 DEBUG [RS_CLOSE_REGION-regionserver/bd6592c37bb2:0-0 {event_type=M_RS_CLOSE_REGION, pid=55}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 63842f73863e9e7b9587486b122e6aae, store=B 2024-11-24T22:14:46,832 DEBUG [RS_CLOSE_REGION-regionserver/bd6592c37bb2:0-0 {event_type=M_RS_CLOSE_REGION, pid=55}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-24T22:14:46,832 DEBUG [RS_CLOSE_REGION-regionserver/bd6592c37bb2:0-0 {event_type=M_RS_CLOSE_REGION, pid=55}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 63842f73863e9e7b9587486b122e6aae, store=C 2024-11-24T22:14:46,832 DEBUG [RS_CLOSE_REGION-regionserver/bd6592c37bb2:0-0 {event_type=M_RS_CLOSE_REGION, pid=55}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-24T22:14:46,839 DEBUG [RS_CLOSE_REGION-regionserver/bd6592c37bb2:0-0 {event_type=M_RS_CLOSE_REGION, pid=55}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411243f41dba08a4a449da306dd6125b8f2ac_63842f73863e9e7b9587486b122e6aae is 50, key is test_row_0/A:col10/1732486484293/Put/seqid=0 2024-11-24T22:14:46,843 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45929 is added to blk_1073742063_1239 (size=12454) 2024-11-24T22:14:47,011 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37701 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=52 2024-11-24T22:14:47,245 DEBUG [RS_CLOSE_REGION-regionserver/bd6592c37bb2:0-0 {event_type=M_RS_CLOSE_REGION, pid=55}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T22:14:47,251 INFO [RS_CLOSE_REGION-regionserver/bd6592c37bb2:0-0 {event_type=M_RS_CLOSE_REGION, pid=55}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411243f41dba08a4a449da306dd6125b8f2ac_63842f73863e9e7b9587486b122e6aae to hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411243f41dba08a4a449da306dd6125b8f2ac_63842f73863e9e7b9587486b122e6aae 2024-11-24T22:14:47,252 DEBUG [RS_CLOSE_REGION-regionserver/bd6592c37bb2:0-0 {event_type=M_RS_CLOSE_REGION, pid=55}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/63842f73863e9e7b9587486b122e6aae/.tmp/A/8710268c8f6f4017b204d10b6ecccc64, store: [table=TestAcidGuarantees family=A region=63842f73863e9e7b9587486b122e6aae] 2024-11-24T22:14:47,253 DEBUG [RS_CLOSE_REGION-regionserver/bd6592c37bb2:0-0 {event_type=M_RS_CLOSE_REGION, pid=55}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/63842f73863e9e7b9587486b122e6aae/.tmp/A/8710268c8f6f4017b204d10b6ecccc64 is 175, key is test_row_0/A:col10/1732486484293/Put/seqid=0 2024-11-24T22:14:47,263 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45929 is added to blk_1073742064_1240 (size=31255) 2024-11-24T22:14:47,264 INFO [RS_CLOSE_REGION-regionserver/bd6592c37bb2:0-0 {event_type=M_RS_CLOSE_REGION, pid=55}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=494, memsize=8.9 K, hasBloomFilter=true, into tmp file hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/63842f73863e9e7b9587486b122e6aae/.tmp/A/8710268c8f6f4017b204d10b6ecccc64 2024-11-24T22:14:47,272 DEBUG [RS_CLOSE_REGION-regionserver/bd6592c37bb2:0-0 {event_type=M_RS_CLOSE_REGION, pid=55}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/63842f73863e9e7b9587486b122e6aae/.tmp/B/ad92a2ba5205468685ba83de56b0905a is 50, key is test_row_0/B:col10/1732486484293/Put/seqid=0 2024-11-24T22:14:47,279 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45929 is added to blk_1073742065_1241 (size=12301) 2024-11-24T22:14:47,521 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37701 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=52 2024-11-24T22:14:47,680 INFO [RS_CLOSE_REGION-regionserver/bd6592c37bb2:0-0 {event_type=M_RS_CLOSE_REGION, pid=55}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=8.95 KB at sequenceid=494 (bloomFilter=true), to=hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/63842f73863e9e7b9587486b122e6aae/.tmp/B/ad92a2ba5205468685ba83de56b0905a 2024-11-24T22:14:47,688 DEBUG [RS_CLOSE_REGION-regionserver/bd6592c37bb2:0-0 {event_type=M_RS_CLOSE_REGION, pid=55}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/63842f73863e9e7b9587486b122e6aae/.tmp/C/161093560a95433a9578b4c558493beb is 50, key is test_row_0/C:col10/1732486484293/Put/seqid=0 2024-11-24T22:14:47,698 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45929 is added to blk_1073742066_1242 (size=12301) 2024-11-24T22:14:48,099 INFO [RS_CLOSE_REGION-regionserver/bd6592c37bb2:0-0 {event_type=M_RS_CLOSE_REGION, pid=55}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=8.95 KB at sequenceid=494 (bloomFilter=true), to=hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/63842f73863e9e7b9587486b122e6aae/.tmp/C/161093560a95433a9578b4c558493beb 2024-11-24T22:14:48,103 DEBUG [RS_CLOSE_REGION-regionserver/bd6592c37bb2:0-0 {event_type=M_RS_CLOSE_REGION, pid=55}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/63842f73863e9e7b9587486b122e6aae/.tmp/A/8710268c8f6f4017b204d10b6ecccc64 as hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/63842f73863e9e7b9587486b122e6aae/A/8710268c8f6f4017b204d10b6ecccc64 2024-11-24T22:14:48,107 INFO [RS_CLOSE_REGION-regionserver/bd6592c37bb2:0-0 {event_type=M_RS_CLOSE_REGION, pid=55}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/63842f73863e9e7b9587486b122e6aae/A/8710268c8f6f4017b204d10b6ecccc64, entries=150, sequenceid=494, filesize=30.5 K 2024-11-24T22:14:48,108 DEBUG [RS_CLOSE_REGION-regionserver/bd6592c37bb2:0-0 {event_type=M_RS_CLOSE_REGION, pid=55}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/63842f73863e9e7b9587486b122e6aae/.tmp/B/ad92a2ba5205468685ba83de56b0905a as hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/63842f73863e9e7b9587486b122e6aae/B/ad92a2ba5205468685ba83de56b0905a 2024-11-24T22:14:48,111 INFO [RS_CLOSE_REGION-regionserver/bd6592c37bb2:0-0 {event_type=M_RS_CLOSE_REGION, pid=55}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/63842f73863e9e7b9587486b122e6aae/B/ad92a2ba5205468685ba83de56b0905a, entries=150, sequenceid=494, filesize=12.0 K 2024-11-24T22:14:48,111 DEBUG [RS_CLOSE_REGION-regionserver/bd6592c37bb2:0-0 {event_type=M_RS_CLOSE_REGION, pid=55}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/63842f73863e9e7b9587486b122e6aae/.tmp/C/161093560a95433a9578b4c558493beb as hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/63842f73863e9e7b9587486b122e6aae/C/161093560a95433a9578b4c558493beb 2024-11-24T22:14:48,114 INFO [RS_CLOSE_REGION-regionserver/bd6592c37bb2:0-0 {event_type=M_RS_CLOSE_REGION, pid=55}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/63842f73863e9e7b9587486b122e6aae/C/161093560a95433a9578b4c558493beb, entries=150, sequenceid=494, filesize=12.0 K 2024-11-24T22:14:48,115 INFO [RS_CLOSE_REGION-regionserver/bd6592c37bb2:0-0 {event_type=M_RS_CLOSE_REGION, pid=55}] regionserver.HRegion(3140): Finished flush of dataSize ~26.84 KB/27480, heapSize ~80.39 KB/82320, currentSize=0 B/0 for 63842f73863e9e7b9587486b122e6aae in 1284ms, sequenceid=494, compaction requested=false 2024-11-24T22:14:48,116 DEBUG [StoreCloser-TestAcidGuarantees,,1732486460633.63842f73863e9e7b9587486b122e6aae.-1 {}] regionserver.HStore(2317): Moving the files [hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/63842f73863e9e7b9587486b122e6aae/A/13672022da204af48915a725fbfa98a5, hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/63842f73863e9e7b9587486b122e6aae/A/fade54e87e5f4b1b93f1630813554439, hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/63842f73863e9e7b9587486b122e6aae/A/58aff1a92da2493ab1bf579d455a3632, hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/63842f73863e9e7b9587486b122e6aae/A/61c5cebf766f4aa09183bcfd524ae066, hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/63842f73863e9e7b9587486b122e6aae/A/f00867b3407d44fa9ab08e3464c78907, hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/63842f73863e9e7b9587486b122e6aae/A/fa7b82d0c1014b23862f5a4f118a9a59, hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/63842f73863e9e7b9587486b122e6aae/A/7a8e21a07dbe4b9f9a240d4f840c9c50, hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/63842f73863e9e7b9587486b122e6aae/A/79994ec69119412bacc781ba6aa897bb, hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/63842f73863e9e7b9587486b122e6aae/A/35f72322d95641559d1e43da1cc0a431, hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/63842f73863e9e7b9587486b122e6aae/A/a9d13b63e2ed4a508c48dec40823e1c0, hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/63842f73863e9e7b9587486b122e6aae/A/a4e17bb33ebb463ea0574c07bb661b48, hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/63842f73863e9e7b9587486b122e6aae/A/54b3fdfd1b984f089b1a2242f0d9d9b2, hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/63842f73863e9e7b9587486b122e6aae/A/937c2f75b1e34c80aa8852008cd4dce4, hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/63842f73863e9e7b9587486b122e6aae/A/1f159989c0a341c9990fdadf792c01a8, hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/63842f73863e9e7b9587486b122e6aae/A/a80b645125754d9299915c8d52dab903, hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/63842f73863e9e7b9587486b122e6aae/A/9563a22f4a1a47adbf57ca6ce108fb23, hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/63842f73863e9e7b9587486b122e6aae/A/cf64aec7f35742de8b3fb00ac564ba08, hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/63842f73863e9e7b9587486b122e6aae/A/ecdf5fa4631f4cc4bfccf8ac391dfc88, hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/63842f73863e9e7b9587486b122e6aae/A/a5c5777a76c04dc7a2b81e81d7e613eb, hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/63842f73863e9e7b9587486b122e6aae/A/4e14986f7371464d829e437f79ceb481, hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/63842f73863e9e7b9587486b122e6aae/A/e4f431b6491e4a06b5c36163eb80cecf, hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/63842f73863e9e7b9587486b122e6aae/A/22daac1eb902430cb4be1109cb6cfa80, hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/63842f73863e9e7b9587486b122e6aae/A/a09a0cf5571f473d87794d3878c6555e, hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/63842f73863e9e7b9587486b122e6aae/A/477957c1db11458f8b66d226b8ee3aeb, hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/63842f73863e9e7b9587486b122e6aae/A/b45983b335d44ecb8c6b81e63ccb78e4, hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/63842f73863e9e7b9587486b122e6aae/A/dd2be7ae7d4045a881cc178f9e22441d, hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/63842f73863e9e7b9587486b122e6aae/A/2b8c21ff057c4163b79d571652f1f38c, hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/63842f73863e9e7b9587486b122e6aae/A/4e39768966df4af1ba07134b48a32c6e, hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/63842f73863e9e7b9587486b122e6aae/A/4fbba21c7791450da1de7eb49457b3fd, hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/63842f73863e9e7b9587486b122e6aae/A/80e746b9efc84951a0532b114e66180a] to archive 2024-11-24T22:14:48,117 DEBUG [StoreCloser-TestAcidGuarantees,,1732486460633.63842f73863e9e7b9587486b122e6aae.-1 {}] backup.HFileArchiver(360): Archiving compacted files. 2024-11-24T22:14:48,118 DEBUG [StoreCloser-TestAcidGuarantees,,1732486460633.63842f73863e9e7b9587486b122e6aae.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/63842f73863e9e7b9587486b122e6aae/A/13672022da204af48915a725fbfa98a5 to hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/archive/data/default/TestAcidGuarantees/63842f73863e9e7b9587486b122e6aae/A/13672022da204af48915a725fbfa98a5 2024-11-24T22:14:48,119 DEBUG [StoreCloser-TestAcidGuarantees,,1732486460633.63842f73863e9e7b9587486b122e6aae.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/63842f73863e9e7b9587486b122e6aae/A/fade54e87e5f4b1b93f1630813554439 to hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/archive/data/default/TestAcidGuarantees/63842f73863e9e7b9587486b122e6aae/A/fade54e87e5f4b1b93f1630813554439 2024-11-24T22:14:48,120 DEBUG [StoreCloser-TestAcidGuarantees,,1732486460633.63842f73863e9e7b9587486b122e6aae.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/63842f73863e9e7b9587486b122e6aae/A/58aff1a92da2493ab1bf579d455a3632 to hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/archive/data/default/TestAcidGuarantees/63842f73863e9e7b9587486b122e6aae/A/58aff1a92da2493ab1bf579d455a3632 2024-11-24T22:14:48,121 DEBUG [StoreCloser-TestAcidGuarantees,,1732486460633.63842f73863e9e7b9587486b122e6aae.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/63842f73863e9e7b9587486b122e6aae/A/61c5cebf766f4aa09183bcfd524ae066 to hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/archive/data/default/TestAcidGuarantees/63842f73863e9e7b9587486b122e6aae/A/61c5cebf766f4aa09183bcfd524ae066 2024-11-24T22:14:48,122 DEBUG [StoreCloser-TestAcidGuarantees,,1732486460633.63842f73863e9e7b9587486b122e6aae.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/63842f73863e9e7b9587486b122e6aae/A/f00867b3407d44fa9ab08e3464c78907 to hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/archive/data/default/TestAcidGuarantees/63842f73863e9e7b9587486b122e6aae/A/f00867b3407d44fa9ab08e3464c78907 2024-11-24T22:14:48,123 DEBUG [StoreCloser-TestAcidGuarantees,,1732486460633.63842f73863e9e7b9587486b122e6aae.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/63842f73863e9e7b9587486b122e6aae/A/fa7b82d0c1014b23862f5a4f118a9a59 to hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/archive/data/default/TestAcidGuarantees/63842f73863e9e7b9587486b122e6aae/A/fa7b82d0c1014b23862f5a4f118a9a59 2024-11-24T22:14:48,124 DEBUG [StoreCloser-TestAcidGuarantees,,1732486460633.63842f73863e9e7b9587486b122e6aae.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/63842f73863e9e7b9587486b122e6aae/A/7a8e21a07dbe4b9f9a240d4f840c9c50 to hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/archive/data/default/TestAcidGuarantees/63842f73863e9e7b9587486b122e6aae/A/7a8e21a07dbe4b9f9a240d4f840c9c50 2024-11-24T22:14:48,125 DEBUG [StoreCloser-TestAcidGuarantees,,1732486460633.63842f73863e9e7b9587486b122e6aae.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/63842f73863e9e7b9587486b122e6aae/A/79994ec69119412bacc781ba6aa897bb to hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/archive/data/default/TestAcidGuarantees/63842f73863e9e7b9587486b122e6aae/A/79994ec69119412bacc781ba6aa897bb 2024-11-24T22:14:48,126 DEBUG [StoreCloser-TestAcidGuarantees,,1732486460633.63842f73863e9e7b9587486b122e6aae.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/63842f73863e9e7b9587486b122e6aae/A/35f72322d95641559d1e43da1cc0a431 to hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/archive/data/default/TestAcidGuarantees/63842f73863e9e7b9587486b122e6aae/A/35f72322d95641559d1e43da1cc0a431 2024-11-24T22:14:48,127 DEBUG [StoreCloser-TestAcidGuarantees,,1732486460633.63842f73863e9e7b9587486b122e6aae.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/63842f73863e9e7b9587486b122e6aae/A/a9d13b63e2ed4a508c48dec40823e1c0 to hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/archive/data/default/TestAcidGuarantees/63842f73863e9e7b9587486b122e6aae/A/a9d13b63e2ed4a508c48dec40823e1c0 2024-11-24T22:14:48,128 DEBUG [StoreCloser-TestAcidGuarantees,,1732486460633.63842f73863e9e7b9587486b122e6aae.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/63842f73863e9e7b9587486b122e6aae/A/a4e17bb33ebb463ea0574c07bb661b48 to hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/archive/data/default/TestAcidGuarantees/63842f73863e9e7b9587486b122e6aae/A/a4e17bb33ebb463ea0574c07bb661b48 2024-11-24T22:14:48,129 DEBUG [StoreCloser-TestAcidGuarantees,,1732486460633.63842f73863e9e7b9587486b122e6aae.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/63842f73863e9e7b9587486b122e6aae/A/54b3fdfd1b984f089b1a2242f0d9d9b2 to hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/archive/data/default/TestAcidGuarantees/63842f73863e9e7b9587486b122e6aae/A/54b3fdfd1b984f089b1a2242f0d9d9b2 2024-11-24T22:14:48,130 DEBUG [StoreCloser-TestAcidGuarantees,,1732486460633.63842f73863e9e7b9587486b122e6aae.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/63842f73863e9e7b9587486b122e6aae/A/937c2f75b1e34c80aa8852008cd4dce4 to hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/archive/data/default/TestAcidGuarantees/63842f73863e9e7b9587486b122e6aae/A/937c2f75b1e34c80aa8852008cd4dce4 2024-11-24T22:14:48,131 DEBUG [StoreCloser-TestAcidGuarantees,,1732486460633.63842f73863e9e7b9587486b122e6aae.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/63842f73863e9e7b9587486b122e6aae/A/1f159989c0a341c9990fdadf792c01a8 to hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/archive/data/default/TestAcidGuarantees/63842f73863e9e7b9587486b122e6aae/A/1f159989c0a341c9990fdadf792c01a8 2024-11-24T22:14:48,131 DEBUG [StoreCloser-TestAcidGuarantees,,1732486460633.63842f73863e9e7b9587486b122e6aae.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/63842f73863e9e7b9587486b122e6aae/A/a80b645125754d9299915c8d52dab903 to hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/archive/data/default/TestAcidGuarantees/63842f73863e9e7b9587486b122e6aae/A/a80b645125754d9299915c8d52dab903 2024-11-24T22:14:48,132 DEBUG [StoreCloser-TestAcidGuarantees,,1732486460633.63842f73863e9e7b9587486b122e6aae.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/63842f73863e9e7b9587486b122e6aae/A/9563a22f4a1a47adbf57ca6ce108fb23 to hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/archive/data/default/TestAcidGuarantees/63842f73863e9e7b9587486b122e6aae/A/9563a22f4a1a47adbf57ca6ce108fb23 2024-11-24T22:14:48,134 DEBUG [StoreCloser-TestAcidGuarantees,,1732486460633.63842f73863e9e7b9587486b122e6aae.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/63842f73863e9e7b9587486b122e6aae/A/cf64aec7f35742de8b3fb00ac564ba08 to hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/archive/data/default/TestAcidGuarantees/63842f73863e9e7b9587486b122e6aae/A/cf64aec7f35742de8b3fb00ac564ba08 2024-11-24T22:14:48,134 DEBUG [StoreCloser-TestAcidGuarantees,,1732486460633.63842f73863e9e7b9587486b122e6aae.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/63842f73863e9e7b9587486b122e6aae/A/ecdf5fa4631f4cc4bfccf8ac391dfc88 to hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/archive/data/default/TestAcidGuarantees/63842f73863e9e7b9587486b122e6aae/A/ecdf5fa4631f4cc4bfccf8ac391dfc88 2024-11-24T22:14:48,135 DEBUG [StoreCloser-TestAcidGuarantees,,1732486460633.63842f73863e9e7b9587486b122e6aae.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/63842f73863e9e7b9587486b122e6aae/A/a5c5777a76c04dc7a2b81e81d7e613eb to hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/archive/data/default/TestAcidGuarantees/63842f73863e9e7b9587486b122e6aae/A/a5c5777a76c04dc7a2b81e81d7e613eb 2024-11-24T22:14:48,136 DEBUG [StoreCloser-TestAcidGuarantees,,1732486460633.63842f73863e9e7b9587486b122e6aae.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/63842f73863e9e7b9587486b122e6aae/A/4e14986f7371464d829e437f79ceb481 to hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/archive/data/default/TestAcidGuarantees/63842f73863e9e7b9587486b122e6aae/A/4e14986f7371464d829e437f79ceb481 2024-11-24T22:14:48,137 DEBUG [StoreCloser-TestAcidGuarantees,,1732486460633.63842f73863e9e7b9587486b122e6aae.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/63842f73863e9e7b9587486b122e6aae/A/e4f431b6491e4a06b5c36163eb80cecf to hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/archive/data/default/TestAcidGuarantees/63842f73863e9e7b9587486b122e6aae/A/e4f431b6491e4a06b5c36163eb80cecf 2024-11-24T22:14:48,138 DEBUG [StoreCloser-TestAcidGuarantees,,1732486460633.63842f73863e9e7b9587486b122e6aae.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/63842f73863e9e7b9587486b122e6aae/A/22daac1eb902430cb4be1109cb6cfa80 to hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/archive/data/default/TestAcidGuarantees/63842f73863e9e7b9587486b122e6aae/A/22daac1eb902430cb4be1109cb6cfa80 2024-11-24T22:14:48,139 DEBUG [StoreCloser-TestAcidGuarantees,,1732486460633.63842f73863e9e7b9587486b122e6aae.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/63842f73863e9e7b9587486b122e6aae/A/a09a0cf5571f473d87794d3878c6555e to hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/archive/data/default/TestAcidGuarantees/63842f73863e9e7b9587486b122e6aae/A/a09a0cf5571f473d87794d3878c6555e 2024-11-24T22:14:48,140 DEBUG [StoreCloser-TestAcidGuarantees,,1732486460633.63842f73863e9e7b9587486b122e6aae.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/63842f73863e9e7b9587486b122e6aae/A/477957c1db11458f8b66d226b8ee3aeb to hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/archive/data/default/TestAcidGuarantees/63842f73863e9e7b9587486b122e6aae/A/477957c1db11458f8b66d226b8ee3aeb 2024-11-24T22:14:48,141 DEBUG [StoreCloser-TestAcidGuarantees,,1732486460633.63842f73863e9e7b9587486b122e6aae.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/63842f73863e9e7b9587486b122e6aae/A/b45983b335d44ecb8c6b81e63ccb78e4 to hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/archive/data/default/TestAcidGuarantees/63842f73863e9e7b9587486b122e6aae/A/b45983b335d44ecb8c6b81e63ccb78e4 2024-11-24T22:14:48,142 DEBUG [StoreCloser-TestAcidGuarantees,,1732486460633.63842f73863e9e7b9587486b122e6aae.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/63842f73863e9e7b9587486b122e6aae/A/dd2be7ae7d4045a881cc178f9e22441d to hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/archive/data/default/TestAcidGuarantees/63842f73863e9e7b9587486b122e6aae/A/dd2be7ae7d4045a881cc178f9e22441d 2024-11-24T22:14:48,143 DEBUG [StoreCloser-TestAcidGuarantees,,1732486460633.63842f73863e9e7b9587486b122e6aae.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/63842f73863e9e7b9587486b122e6aae/A/2b8c21ff057c4163b79d571652f1f38c to hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/archive/data/default/TestAcidGuarantees/63842f73863e9e7b9587486b122e6aae/A/2b8c21ff057c4163b79d571652f1f38c 2024-11-24T22:14:48,144 DEBUG [StoreCloser-TestAcidGuarantees,,1732486460633.63842f73863e9e7b9587486b122e6aae.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/63842f73863e9e7b9587486b122e6aae/A/4e39768966df4af1ba07134b48a32c6e to hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/archive/data/default/TestAcidGuarantees/63842f73863e9e7b9587486b122e6aae/A/4e39768966df4af1ba07134b48a32c6e 2024-11-24T22:14:48,145 DEBUG [StoreCloser-TestAcidGuarantees,,1732486460633.63842f73863e9e7b9587486b122e6aae.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/63842f73863e9e7b9587486b122e6aae/A/4fbba21c7791450da1de7eb49457b3fd to hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/archive/data/default/TestAcidGuarantees/63842f73863e9e7b9587486b122e6aae/A/4fbba21c7791450da1de7eb49457b3fd 2024-11-24T22:14:48,146 DEBUG [StoreCloser-TestAcidGuarantees,,1732486460633.63842f73863e9e7b9587486b122e6aae.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/63842f73863e9e7b9587486b122e6aae/A/80e746b9efc84951a0532b114e66180a to hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/archive/data/default/TestAcidGuarantees/63842f73863e9e7b9587486b122e6aae/A/80e746b9efc84951a0532b114e66180a 2024-11-24T22:14:48,147 DEBUG [StoreCloser-TestAcidGuarantees,,1732486460633.63842f73863e9e7b9587486b122e6aae.-1 {}] regionserver.HStore(2317): Moving the files [hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/63842f73863e9e7b9587486b122e6aae/B/e0d8bb8944b64909b312bea8b1cacdd8, hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/63842f73863e9e7b9587486b122e6aae/B/02bd6ba0eecd48b99e5ad15a050a2011, hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/63842f73863e9e7b9587486b122e6aae/B/b4285931220446c38584c66287d3a5b9, hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/63842f73863e9e7b9587486b122e6aae/B/026af172cb4249039e2f443f841ab798, hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/63842f73863e9e7b9587486b122e6aae/B/e27b972d2a874d369095cb09a68a3f9f, hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/63842f73863e9e7b9587486b122e6aae/B/85d047d5e1164e14a0ecf07a70e8146c, hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/63842f73863e9e7b9587486b122e6aae/B/df8f192c6e7149afb3fabf9e6698e187, hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/63842f73863e9e7b9587486b122e6aae/B/5eda402104654194b9e33e227dc03d50, hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/63842f73863e9e7b9587486b122e6aae/B/7b7482699d6743d1874a0762ec11ae0e, hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/63842f73863e9e7b9587486b122e6aae/B/ed25299c01a145d88eaa3f03fd0b147e, hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/63842f73863e9e7b9587486b122e6aae/B/6b1102e285ca494e9044ea4767549dc0, hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/63842f73863e9e7b9587486b122e6aae/B/8546de4e512a4074a0c26e2f30373177, hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/63842f73863e9e7b9587486b122e6aae/B/7a02ad10845946b78864d3662caaedbd, hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/63842f73863e9e7b9587486b122e6aae/B/df1cfca44f444160a40eb44a80dd877f, hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/63842f73863e9e7b9587486b122e6aae/B/80c3be87a7bc406abcd49b8ff894561f, hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/63842f73863e9e7b9587486b122e6aae/B/455dcb4f2aca48b98940fe2ebddff3fb, hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/63842f73863e9e7b9587486b122e6aae/B/fd1b77c22a4c4a79a625caa3d88bbeb4, hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/63842f73863e9e7b9587486b122e6aae/B/81e2cda377e2474c933a28ab8f351261, hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/63842f73863e9e7b9587486b122e6aae/B/09eff7dc8cda475b93df975f70ffcc4a, hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/63842f73863e9e7b9587486b122e6aae/B/c239f7036c684af48076a9457f4f24aa, hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/63842f73863e9e7b9587486b122e6aae/B/a4f2a2e251974d68900f71c211d0d76c, hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/63842f73863e9e7b9587486b122e6aae/B/e2d876c1c5b849819f4edeeba50d9ad3, hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/63842f73863e9e7b9587486b122e6aae/B/48419645ce0442d981fa746ee68828e1, hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/63842f73863e9e7b9587486b122e6aae/B/053a596b3ce94469af401422641d7f60, hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/63842f73863e9e7b9587486b122e6aae/B/916c780a9f1c48a7a780b2aa308cc2e3, hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/63842f73863e9e7b9587486b122e6aae/B/0be02238389c498580e6164758290ed8, hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/63842f73863e9e7b9587486b122e6aae/B/1e6412e76d09485b8b41413a341073af, hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/63842f73863e9e7b9587486b122e6aae/B/93cf1c13def944adb5dbfacdde2a8cb0, hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/63842f73863e9e7b9587486b122e6aae/B/db681ed797f14c9092acfa530ecefd95, hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/63842f73863e9e7b9587486b122e6aae/B/7c235b3244fb42a080db1fba055d1237] to archive 2024-11-24T22:14:48,148 DEBUG [StoreCloser-TestAcidGuarantees,,1732486460633.63842f73863e9e7b9587486b122e6aae.-1 {}] backup.HFileArchiver(360): Archiving compacted files. 2024-11-24T22:14:48,149 DEBUG [StoreCloser-TestAcidGuarantees,,1732486460633.63842f73863e9e7b9587486b122e6aae.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/63842f73863e9e7b9587486b122e6aae/B/e0d8bb8944b64909b312bea8b1cacdd8 to hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/archive/data/default/TestAcidGuarantees/63842f73863e9e7b9587486b122e6aae/B/e0d8bb8944b64909b312bea8b1cacdd8 2024-11-24T22:14:48,150 DEBUG [StoreCloser-TestAcidGuarantees,,1732486460633.63842f73863e9e7b9587486b122e6aae.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/63842f73863e9e7b9587486b122e6aae/B/02bd6ba0eecd48b99e5ad15a050a2011 to hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/archive/data/default/TestAcidGuarantees/63842f73863e9e7b9587486b122e6aae/B/02bd6ba0eecd48b99e5ad15a050a2011 2024-11-24T22:14:48,151 DEBUG [StoreCloser-TestAcidGuarantees,,1732486460633.63842f73863e9e7b9587486b122e6aae.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/63842f73863e9e7b9587486b122e6aae/B/b4285931220446c38584c66287d3a5b9 to hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/archive/data/default/TestAcidGuarantees/63842f73863e9e7b9587486b122e6aae/B/b4285931220446c38584c66287d3a5b9 2024-11-24T22:14:48,152 DEBUG [StoreCloser-TestAcidGuarantees,,1732486460633.63842f73863e9e7b9587486b122e6aae.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/63842f73863e9e7b9587486b122e6aae/B/026af172cb4249039e2f443f841ab798 to hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/archive/data/default/TestAcidGuarantees/63842f73863e9e7b9587486b122e6aae/B/026af172cb4249039e2f443f841ab798 2024-11-24T22:14:48,152 DEBUG [StoreCloser-TestAcidGuarantees,,1732486460633.63842f73863e9e7b9587486b122e6aae.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/63842f73863e9e7b9587486b122e6aae/B/e27b972d2a874d369095cb09a68a3f9f to hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/archive/data/default/TestAcidGuarantees/63842f73863e9e7b9587486b122e6aae/B/e27b972d2a874d369095cb09a68a3f9f 2024-11-24T22:14:48,153 DEBUG [StoreCloser-TestAcidGuarantees,,1732486460633.63842f73863e9e7b9587486b122e6aae.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/63842f73863e9e7b9587486b122e6aae/B/85d047d5e1164e14a0ecf07a70e8146c to hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/archive/data/default/TestAcidGuarantees/63842f73863e9e7b9587486b122e6aae/B/85d047d5e1164e14a0ecf07a70e8146c 2024-11-24T22:14:48,154 DEBUG [StoreCloser-TestAcidGuarantees,,1732486460633.63842f73863e9e7b9587486b122e6aae.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/63842f73863e9e7b9587486b122e6aae/B/df8f192c6e7149afb3fabf9e6698e187 to hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/archive/data/default/TestAcidGuarantees/63842f73863e9e7b9587486b122e6aae/B/df8f192c6e7149afb3fabf9e6698e187 2024-11-24T22:14:48,155 DEBUG [StoreCloser-TestAcidGuarantees,,1732486460633.63842f73863e9e7b9587486b122e6aae.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/63842f73863e9e7b9587486b122e6aae/B/5eda402104654194b9e33e227dc03d50 to hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/archive/data/default/TestAcidGuarantees/63842f73863e9e7b9587486b122e6aae/B/5eda402104654194b9e33e227dc03d50 2024-11-24T22:14:48,156 DEBUG [StoreCloser-TestAcidGuarantees,,1732486460633.63842f73863e9e7b9587486b122e6aae.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/63842f73863e9e7b9587486b122e6aae/B/7b7482699d6743d1874a0762ec11ae0e to hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/archive/data/default/TestAcidGuarantees/63842f73863e9e7b9587486b122e6aae/B/7b7482699d6743d1874a0762ec11ae0e 2024-11-24T22:14:48,157 DEBUG [StoreCloser-TestAcidGuarantees,,1732486460633.63842f73863e9e7b9587486b122e6aae.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/63842f73863e9e7b9587486b122e6aae/B/ed25299c01a145d88eaa3f03fd0b147e to hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/archive/data/default/TestAcidGuarantees/63842f73863e9e7b9587486b122e6aae/B/ed25299c01a145d88eaa3f03fd0b147e 2024-11-24T22:14:48,158 DEBUG [StoreCloser-TestAcidGuarantees,,1732486460633.63842f73863e9e7b9587486b122e6aae.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/63842f73863e9e7b9587486b122e6aae/B/6b1102e285ca494e9044ea4767549dc0 to hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/archive/data/default/TestAcidGuarantees/63842f73863e9e7b9587486b122e6aae/B/6b1102e285ca494e9044ea4767549dc0 2024-11-24T22:14:48,158 DEBUG [StoreCloser-TestAcidGuarantees,,1732486460633.63842f73863e9e7b9587486b122e6aae.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/63842f73863e9e7b9587486b122e6aae/B/8546de4e512a4074a0c26e2f30373177 to hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/archive/data/default/TestAcidGuarantees/63842f73863e9e7b9587486b122e6aae/B/8546de4e512a4074a0c26e2f30373177 2024-11-24T22:14:48,159 DEBUG [StoreCloser-TestAcidGuarantees,,1732486460633.63842f73863e9e7b9587486b122e6aae.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/63842f73863e9e7b9587486b122e6aae/B/7a02ad10845946b78864d3662caaedbd to hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/archive/data/default/TestAcidGuarantees/63842f73863e9e7b9587486b122e6aae/B/7a02ad10845946b78864d3662caaedbd 2024-11-24T22:14:48,160 DEBUG [StoreCloser-TestAcidGuarantees,,1732486460633.63842f73863e9e7b9587486b122e6aae.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/63842f73863e9e7b9587486b122e6aae/B/df1cfca44f444160a40eb44a80dd877f to hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/archive/data/default/TestAcidGuarantees/63842f73863e9e7b9587486b122e6aae/B/df1cfca44f444160a40eb44a80dd877f 2024-11-24T22:14:48,161 DEBUG [StoreCloser-TestAcidGuarantees,,1732486460633.63842f73863e9e7b9587486b122e6aae.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/63842f73863e9e7b9587486b122e6aae/B/80c3be87a7bc406abcd49b8ff894561f to hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/archive/data/default/TestAcidGuarantees/63842f73863e9e7b9587486b122e6aae/B/80c3be87a7bc406abcd49b8ff894561f 2024-11-24T22:14:48,162 DEBUG [StoreCloser-TestAcidGuarantees,,1732486460633.63842f73863e9e7b9587486b122e6aae.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/63842f73863e9e7b9587486b122e6aae/B/455dcb4f2aca48b98940fe2ebddff3fb to hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/archive/data/default/TestAcidGuarantees/63842f73863e9e7b9587486b122e6aae/B/455dcb4f2aca48b98940fe2ebddff3fb 2024-11-24T22:14:48,163 DEBUG [StoreCloser-TestAcidGuarantees,,1732486460633.63842f73863e9e7b9587486b122e6aae.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/63842f73863e9e7b9587486b122e6aae/B/fd1b77c22a4c4a79a625caa3d88bbeb4 to hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/archive/data/default/TestAcidGuarantees/63842f73863e9e7b9587486b122e6aae/B/fd1b77c22a4c4a79a625caa3d88bbeb4 2024-11-24T22:14:48,164 DEBUG [StoreCloser-TestAcidGuarantees,,1732486460633.63842f73863e9e7b9587486b122e6aae.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/63842f73863e9e7b9587486b122e6aae/B/81e2cda377e2474c933a28ab8f351261 to hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/archive/data/default/TestAcidGuarantees/63842f73863e9e7b9587486b122e6aae/B/81e2cda377e2474c933a28ab8f351261 2024-11-24T22:14:48,165 DEBUG [StoreCloser-TestAcidGuarantees,,1732486460633.63842f73863e9e7b9587486b122e6aae.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/63842f73863e9e7b9587486b122e6aae/B/09eff7dc8cda475b93df975f70ffcc4a to hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/archive/data/default/TestAcidGuarantees/63842f73863e9e7b9587486b122e6aae/B/09eff7dc8cda475b93df975f70ffcc4a 2024-11-24T22:14:48,166 DEBUG [StoreCloser-TestAcidGuarantees,,1732486460633.63842f73863e9e7b9587486b122e6aae.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/63842f73863e9e7b9587486b122e6aae/B/c239f7036c684af48076a9457f4f24aa to hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/archive/data/default/TestAcidGuarantees/63842f73863e9e7b9587486b122e6aae/B/c239f7036c684af48076a9457f4f24aa 2024-11-24T22:14:48,167 DEBUG [StoreCloser-TestAcidGuarantees,,1732486460633.63842f73863e9e7b9587486b122e6aae.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/63842f73863e9e7b9587486b122e6aae/B/a4f2a2e251974d68900f71c211d0d76c to hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/archive/data/default/TestAcidGuarantees/63842f73863e9e7b9587486b122e6aae/B/a4f2a2e251974d68900f71c211d0d76c 2024-11-24T22:14:48,168 DEBUG [StoreCloser-TestAcidGuarantees,,1732486460633.63842f73863e9e7b9587486b122e6aae.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/63842f73863e9e7b9587486b122e6aae/B/e2d876c1c5b849819f4edeeba50d9ad3 to hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/archive/data/default/TestAcidGuarantees/63842f73863e9e7b9587486b122e6aae/B/e2d876c1c5b849819f4edeeba50d9ad3 2024-11-24T22:14:48,169 DEBUG [StoreCloser-TestAcidGuarantees,,1732486460633.63842f73863e9e7b9587486b122e6aae.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/63842f73863e9e7b9587486b122e6aae/B/48419645ce0442d981fa746ee68828e1 to hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/archive/data/default/TestAcidGuarantees/63842f73863e9e7b9587486b122e6aae/B/48419645ce0442d981fa746ee68828e1 2024-11-24T22:14:48,170 DEBUG [StoreCloser-TestAcidGuarantees,,1732486460633.63842f73863e9e7b9587486b122e6aae.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/63842f73863e9e7b9587486b122e6aae/B/053a596b3ce94469af401422641d7f60 to hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/archive/data/default/TestAcidGuarantees/63842f73863e9e7b9587486b122e6aae/B/053a596b3ce94469af401422641d7f60 2024-11-24T22:14:48,171 DEBUG [StoreCloser-TestAcidGuarantees,,1732486460633.63842f73863e9e7b9587486b122e6aae.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/63842f73863e9e7b9587486b122e6aae/B/916c780a9f1c48a7a780b2aa308cc2e3 to hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/archive/data/default/TestAcidGuarantees/63842f73863e9e7b9587486b122e6aae/B/916c780a9f1c48a7a780b2aa308cc2e3 2024-11-24T22:14:48,172 DEBUG [StoreCloser-TestAcidGuarantees,,1732486460633.63842f73863e9e7b9587486b122e6aae.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/63842f73863e9e7b9587486b122e6aae/B/0be02238389c498580e6164758290ed8 to hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/archive/data/default/TestAcidGuarantees/63842f73863e9e7b9587486b122e6aae/B/0be02238389c498580e6164758290ed8 2024-11-24T22:14:48,173 DEBUG [StoreCloser-TestAcidGuarantees,,1732486460633.63842f73863e9e7b9587486b122e6aae.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/63842f73863e9e7b9587486b122e6aae/B/1e6412e76d09485b8b41413a341073af to hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/archive/data/default/TestAcidGuarantees/63842f73863e9e7b9587486b122e6aae/B/1e6412e76d09485b8b41413a341073af 2024-11-24T22:14:48,174 DEBUG [StoreCloser-TestAcidGuarantees,,1732486460633.63842f73863e9e7b9587486b122e6aae.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/63842f73863e9e7b9587486b122e6aae/B/93cf1c13def944adb5dbfacdde2a8cb0 to hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/archive/data/default/TestAcidGuarantees/63842f73863e9e7b9587486b122e6aae/B/93cf1c13def944adb5dbfacdde2a8cb0 2024-11-24T22:14:48,174 DEBUG [StoreCloser-TestAcidGuarantees,,1732486460633.63842f73863e9e7b9587486b122e6aae.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/63842f73863e9e7b9587486b122e6aae/B/db681ed797f14c9092acfa530ecefd95 to hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/archive/data/default/TestAcidGuarantees/63842f73863e9e7b9587486b122e6aae/B/db681ed797f14c9092acfa530ecefd95 2024-11-24T22:14:48,175 DEBUG [StoreCloser-TestAcidGuarantees,,1732486460633.63842f73863e9e7b9587486b122e6aae.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/63842f73863e9e7b9587486b122e6aae/B/7c235b3244fb42a080db1fba055d1237 to hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/archive/data/default/TestAcidGuarantees/63842f73863e9e7b9587486b122e6aae/B/7c235b3244fb42a080db1fba055d1237 2024-11-24T22:14:48,177 DEBUG [StoreCloser-TestAcidGuarantees,,1732486460633.63842f73863e9e7b9587486b122e6aae.-1 {}] regionserver.HStore(2317): Moving the files [hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/63842f73863e9e7b9587486b122e6aae/C/6711daa1d8ff424e8ac2f9cd4c7287f4, hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/63842f73863e9e7b9587486b122e6aae/C/f967706d7c7a4b2aa41d0ce3d33ee73e, hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/63842f73863e9e7b9587486b122e6aae/C/c3ebf3de6c6240cdb34d8d2cd189aadf, hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/63842f73863e9e7b9587486b122e6aae/C/2e9f8709e5ad4ad5a5f86d4aa09c9c61, hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/63842f73863e9e7b9587486b122e6aae/C/6897f56dbe744713beb5afa1e154ea02, hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/63842f73863e9e7b9587486b122e6aae/C/827f6af5e50f48edb4d996590552ae1d, hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/63842f73863e9e7b9587486b122e6aae/C/17481ead002944a3b87ffd04ccafc446, hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/63842f73863e9e7b9587486b122e6aae/C/6942817daf0640c5b53d955de9391e7c, hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/63842f73863e9e7b9587486b122e6aae/C/4d52a0c91d794d46956f4bd1629386f4, hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/63842f73863e9e7b9587486b122e6aae/C/89494cbdc6fb4d5da656158a799b399e, hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/63842f73863e9e7b9587486b122e6aae/C/222b20f7e0474cf6b7bdaea5d17b20e2, hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/63842f73863e9e7b9587486b122e6aae/C/5c1e349f365f4cfda43b21302f209dec, hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/63842f73863e9e7b9587486b122e6aae/C/4923864c6fda45d6b701d9e800bd4c46, hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/63842f73863e9e7b9587486b122e6aae/C/9d3c116f95e046b89d2b1d00eae6999d, hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/63842f73863e9e7b9587486b122e6aae/C/58561c0c991f467ca37137ea8d0cf3df, hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/63842f73863e9e7b9587486b122e6aae/C/17307ba8f7ec483e8fc9c7d11b47d475, hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/63842f73863e9e7b9587486b122e6aae/C/f9001c2421d641e1a3a5fa501a7bf407, hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/63842f73863e9e7b9587486b122e6aae/C/c0250f2ef6c147c1ac6f1adc8c0836d6, hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/63842f73863e9e7b9587486b122e6aae/C/253f3fb757454dfcbb4d4a532093bc69, hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/63842f73863e9e7b9587486b122e6aae/C/38ae68e5da07426fa5ecf7872ef45fed, hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/63842f73863e9e7b9587486b122e6aae/C/7fcba68ad79e4eb7b261b7b5aad297e5, hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/63842f73863e9e7b9587486b122e6aae/C/c9bc933b9cc242a4a268f760548c19e2, hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/63842f73863e9e7b9587486b122e6aae/C/e2065151f0e24dcdada181dbbfc3e7f8, hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/63842f73863e9e7b9587486b122e6aae/C/07bc2a76178f433fb1190a4acc0ae53c, hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/63842f73863e9e7b9587486b122e6aae/C/f0e1b796e92c45918e07acbffa3c8650, hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/63842f73863e9e7b9587486b122e6aae/C/0502b36d90f4444c9da4fb69ebf63c97, hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/63842f73863e9e7b9587486b122e6aae/C/1730f0d31ed04656b27dbffd7e1537c6, hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/63842f73863e9e7b9587486b122e6aae/C/a8fd58c0130242109584ecd578f8cc8c, hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/63842f73863e9e7b9587486b122e6aae/C/46ae7d5ce9c54d38a636ba8fde779f36, hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/63842f73863e9e7b9587486b122e6aae/C/9432eb988e004ad091e33fa3661cc3f0] to archive 2024-11-24T22:14:48,178 DEBUG [StoreCloser-TestAcidGuarantees,,1732486460633.63842f73863e9e7b9587486b122e6aae.-1 {}] backup.HFileArchiver(360): Archiving compacted files. 2024-11-24T22:14:48,179 DEBUG [StoreCloser-TestAcidGuarantees,,1732486460633.63842f73863e9e7b9587486b122e6aae.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/63842f73863e9e7b9587486b122e6aae/C/6711daa1d8ff424e8ac2f9cd4c7287f4 to hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/archive/data/default/TestAcidGuarantees/63842f73863e9e7b9587486b122e6aae/C/6711daa1d8ff424e8ac2f9cd4c7287f4 2024-11-24T22:14:48,180 DEBUG [StoreCloser-TestAcidGuarantees,,1732486460633.63842f73863e9e7b9587486b122e6aae.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/63842f73863e9e7b9587486b122e6aae/C/f967706d7c7a4b2aa41d0ce3d33ee73e to hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/archive/data/default/TestAcidGuarantees/63842f73863e9e7b9587486b122e6aae/C/f967706d7c7a4b2aa41d0ce3d33ee73e 2024-11-24T22:14:48,181 DEBUG [StoreCloser-TestAcidGuarantees,,1732486460633.63842f73863e9e7b9587486b122e6aae.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/63842f73863e9e7b9587486b122e6aae/C/c3ebf3de6c6240cdb34d8d2cd189aadf to hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/archive/data/default/TestAcidGuarantees/63842f73863e9e7b9587486b122e6aae/C/c3ebf3de6c6240cdb34d8d2cd189aadf 2024-11-24T22:14:48,182 DEBUG [StoreCloser-TestAcidGuarantees,,1732486460633.63842f73863e9e7b9587486b122e6aae.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/63842f73863e9e7b9587486b122e6aae/C/2e9f8709e5ad4ad5a5f86d4aa09c9c61 to hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/archive/data/default/TestAcidGuarantees/63842f73863e9e7b9587486b122e6aae/C/2e9f8709e5ad4ad5a5f86d4aa09c9c61 2024-11-24T22:14:48,183 DEBUG [StoreCloser-TestAcidGuarantees,,1732486460633.63842f73863e9e7b9587486b122e6aae.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/63842f73863e9e7b9587486b122e6aae/C/6897f56dbe744713beb5afa1e154ea02 to hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/archive/data/default/TestAcidGuarantees/63842f73863e9e7b9587486b122e6aae/C/6897f56dbe744713beb5afa1e154ea02 2024-11-24T22:14:48,184 DEBUG [StoreCloser-TestAcidGuarantees,,1732486460633.63842f73863e9e7b9587486b122e6aae.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/63842f73863e9e7b9587486b122e6aae/C/827f6af5e50f48edb4d996590552ae1d to hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/archive/data/default/TestAcidGuarantees/63842f73863e9e7b9587486b122e6aae/C/827f6af5e50f48edb4d996590552ae1d 2024-11-24T22:14:48,185 DEBUG [StoreCloser-TestAcidGuarantees,,1732486460633.63842f73863e9e7b9587486b122e6aae.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/63842f73863e9e7b9587486b122e6aae/C/17481ead002944a3b87ffd04ccafc446 to hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/archive/data/default/TestAcidGuarantees/63842f73863e9e7b9587486b122e6aae/C/17481ead002944a3b87ffd04ccafc446 2024-11-24T22:14:48,186 DEBUG [StoreCloser-TestAcidGuarantees,,1732486460633.63842f73863e9e7b9587486b122e6aae.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/63842f73863e9e7b9587486b122e6aae/C/6942817daf0640c5b53d955de9391e7c to hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/archive/data/default/TestAcidGuarantees/63842f73863e9e7b9587486b122e6aae/C/6942817daf0640c5b53d955de9391e7c 2024-11-24T22:14:48,187 DEBUG [StoreCloser-TestAcidGuarantees,,1732486460633.63842f73863e9e7b9587486b122e6aae.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/63842f73863e9e7b9587486b122e6aae/C/4d52a0c91d794d46956f4bd1629386f4 to hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/archive/data/default/TestAcidGuarantees/63842f73863e9e7b9587486b122e6aae/C/4d52a0c91d794d46956f4bd1629386f4 2024-11-24T22:14:48,188 DEBUG [StoreCloser-TestAcidGuarantees,,1732486460633.63842f73863e9e7b9587486b122e6aae.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/63842f73863e9e7b9587486b122e6aae/C/89494cbdc6fb4d5da656158a799b399e to hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/archive/data/default/TestAcidGuarantees/63842f73863e9e7b9587486b122e6aae/C/89494cbdc6fb4d5da656158a799b399e 2024-11-24T22:14:48,190 DEBUG [StoreCloser-TestAcidGuarantees,,1732486460633.63842f73863e9e7b9587486b122e6aae.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/63842f73863e9e7b9587486b122e6aae/C/222b20f7e0474cf6b7bdaea5d17b20e2 to hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/archive/data/default/TestAcidGuarantees/63842f73863e9e7b9587486b122e6aae/C/222b20f7e0474cf6b7bdaea5d17b20e2 2024-11-24T22:14:48,191 DEBUG [StoreCloser-TestAcidGuarantees,,1732486460633.63842f73863e9e7b9587486b122e6aae.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/63842f73863e9e7b9587486b122e6aae/C/5c1e349f365f4cfda43b21302f209dec to hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/archive/data/default/TestAcidGuarantees/63842f73863e9e7b9587486b122e6aae/C/5c1e349f365f4cfda43b21302f209dec 2024-11-24T22:14:48,192 DEBUG [StoreCloser-TestAcidGuarantees,,1732486460633.63842f73863e9e7b9587486b122e6aae.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/63842f73863e9e7b9587486b122e6aae/C/4923864c6fda45d6b701d9e800bd4c46 to hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/archive/data/default/TestAcidGuarantees/63842f73863e9e7b9587486b122e6aae/C/4923864c6fda45d6b701d9e800bd4c46 2024-11-24T22:14:48,193 DEBUG [StoreCloser-TestAcidGuarantees,,1732486460633.63842f73863e9e7b9587486b122e6aae.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/63842f73863e9e7b9587486b122e6aae/C/9d3c116f95e046b89d2b1d00eae6999d to hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/archive/data/default/TestAcidGuarantees/63842f73863e9e7b9587486b122e6aae/C/9d3c116f95e046b89d2b1d00eae6999d 2024-11-24T22:14:48,194 DEBUG [StoreCloser-TestAcidGuarantees,,1732486460633.63842f73863e9e7b9587486b122e6aae.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/63842f73863e9e7b9587486b122e6aae/C/58561c0c991f467ca37137ea8d0cf3df to hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/archive/data/default/TestAcidGuarantees/63842f73863e9e7b9587486b122e6aae/C/58561c0c991f467ca37137ea8d0cf3df 2024-11-24T22:14:48,195 DEBUG [StoreCloser-TestAcidGuarantees,,1732486460633.63842f73863e9e7b9587486b122e6aae.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/63842f73863e9e7b9587486b122e6aae/C/17307ba8f7ec483e8fc9c7d11b47d475 to hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/archive/data/default/TestAcidGuarantees/63842f73863e9e7b9587486b122e6aae/C/17307ba8f7ec483e8fc9c7d11b47d475 2024-11-24T22:14:48,196 DEBUG [StoreCloser-TestAcidGuarantees,,1732486460633.63842f73863e9e7b9587486b122e6aae.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/63842f73863e9e7b9587486b122e6aae/C/f9001c2421d641e1a3a5fa501a7bf407 to hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/archive/data/default/TestAcidGuarantees/63842f73863e9e7b9587486b122e6aae/C/f9001c2421d641e1a3a5fa501a7bf407 2024-11-24T22:14:48,197 DEBUG [StoreCloser-TestAcidGuarantees,,1732486460633.63842f73863e9e7b9587486b122e6aae.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/63842f73863e9e7b9587486b122e6aae/C/c0250f2ef6c147c1ac6f1adc8c0836d6 to hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/archive/data/default/TestAcidGuarantees/63842f73863e9e7b9587486b122e6aae/C/c0250f2ef6c147c1ac6f1adc8c0836d6 2024-11-24T22:14:48,198 DEBUG [StoreCloser-TestAcidGuarantees,,1732486460633.63842f73863e9e7b9587486b122e6aae.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/63842f73863e9e7b9587486b122e6aae/C/253f3fb757454dfcbb4d4a532093bc69 to hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/archive/data/default/TestAcidGuarantees/63842f73863e9e7b9587486b122e6aae/C/253f3fb757454dfcbb4d4a532093bc69 2024-11-24T22:14:48,199 DEBUG [StoreCloser-TestAcidGuarantees,,1732486460633.63842f73863e9e7b9587486b122e6aae.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/63842f73863e9e7b9587486b122e6aae/C/38ae68e5da07426fa5ecf7872ef45fed to hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/archive/data/default/TestAcidGuarantees/63842f73863e9e7b9587486b122e6aae/C/38ae68e5da07426fa5ecf7872ef45fed 2024-11-24T22:14:48,199 DEBUG [StoreCloser-TestAcidGuarantees,,1732486460633.63842f73863e9e7b9587486b122e6aae.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/63842f73863e9e7b9587486b122e6aae/C/7fcba68ad79e4eb7b261b7b5aad297e5 to hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/archive/data/default/TestAcidGuarantees/63842f73863e9e7b9587486b122e6aae/C/7fcba68ad79e4eb7b261b7b5aad297e5 2024-11-24T22:14:48,200 DEBUG [StoreCloser-TestAcidGuarantees,,1732486460633.63842f73863e9e7b9587486b122e6aae.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/63842f73863e9e7b9587486b122e6aae/C/c9bc933b9cc242a4a268f760548c19e2 to hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/archive/data/default/TestAcidGuarantees/63842f73863e9e7b9587486b122e6aae/C/c9bc933b9cc242a4a268f760548c19e2 2024-11-24T22:14:48,201 DEBUG [StoreCloser-TestAcidGuarantees,,1732486460633.63842f73863e9e7b9587486b122e6aae.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/63842f73863e9e7b9587486b122e6aae/C/e2065151f0e24dcdada181dbbfc3e7f8 to hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/archive/data/default/TestAcidGuarantees/63842f73863e9e7b9587486b122e6aae/C/e2065151f0e24dcdada181dbbfc3e7f8 2024-11-24T22:14:48,202 DEBUG [StoreCloser-TestAcidGuarantees,,1732486460633.63842f73863e9e7b9587486b122e6aae.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/63842f73863e9e7b9587486b122e6aae/C/07bc2a76178f433fb1190a4acc0ae53c to hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/archive/data/default/TestAcidGuarantees/63842f73863e9e7b9587486b122e6aae/C/07bc2a76178f433fb1190a4acc0ae53c 2024-11-24T22:14:48,203 DEBUG [StoreCloser-TestAcidGuarantees,,1732486460633.63842f73863e9e7b9587486b122e6aae.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/63842f73863e9e7b9587486b122e6aae/C/f0e1b796e92c45918e07acbffa3c8650 to hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/archive/data/default/TestAcidGuarantees/63842f73863e9e7b9587486b122e6aae/C/f0e1b796e92c45918e07acbffa3c8650 2024-11-24T22:14:48,204 DEBUG [StoreCloser-TestAcidGuarantees,,1732486460633.63842f73863e9e7b9587486b122e6aae.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/63842f73863e9e7b9587486b122e6aae/C/0502b36d90f4444c9da4fb69ebf63c97 to hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/archive/data/default/TestAcidGuarantees/63842f73863e9e7b9587486b122e6aae/C/0502b36d90f4444c9da4fb69ebf63c97 2024-11-24T22:14:48,205 DEBUG [StoreCloser-TestAcidGuarantees,,1732486460633.63842f73863e9e7b9587486b122e6aae.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/63842f73863e9e7b9587486b122e6aae/C/1730f0d31ed04656b27dbffd7e1537c6 to hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/archive/data/default/TestAcidGuarantees/63842f73863e9e7b9587486b122e6aae/C/1730f0d31ed04656b27dbffd7e1537c6 2024-11-24T22:14:48,206 DEBUG [StoreCloser-TestAcidGuarantees,,1732486460633.63842f73863e9e7b9587486b122e6aae.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/63842f73863e9e7b9587486b122e6aae/C/a8fd58c0130242109584ecd578f8cc8c to hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/archive/data/default/TestAcidGuarantees/63842f73863e9e7b9587486b122e6aae/C/a8fd58c0130242109584ecd578f8cc8c 2024-11-24T22:14:48,207 DEBUG [StoreCloser-TestAcidGuarantees,,1732486460633.63842f73863e9e7b9587486b122e6aae.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/63842f73863e9e7b9587486b122e6aae/C/46ae7d5ce9c54d38a636ba8fde779f36 to hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/archive/data/default/TestAcidGuarantees/63842f73863e9e7b9587486b122e6aae/C/46ae7d5ce9c54d38a636ba8fde779f36 2024-11-24T22:14:48,208 DEBUG [StoreCloser-TestAcidGuarantees,,1732486460633.63842f73863e9e7b9587486b122e6aae.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/63842f73863e9e7b9587486b122e6aae/C/9432eb988e004ad091e33fa3661cc3f0 to hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/archive/data/default/TestAcidGuarantees/63842f73863e9e7b9587486b122e6aae/C/9432eb988e004ad091e33fa3661cc3f0 2024-11-24T22:14:48,212 DEBUG [RS_CLOSE_REGION-regionserver/bd6592c37bb2:0-0 {event_type=M_RS_CLOSE_REGION, pid=55}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/63842f73863e9e7b9587486b122e6aae/recovered.edits/497.seqid, newMaxSeqId=497, maxSeqId=4 2024-11-24T22:14:48,213 INFO [RS_CLOSE_REGION-regionserver/bd6592c37bb2:0-0 {event_type=M_RS_CLOSE_REGION, pid=55}] regionserver.HRegion(1973): Closed TestAcidGuarantees,,1732486460633.63842f73863e9e7b9587486b122e6aae. 2024-11-24T22:14:48,213 DEBUG [RS_CLOSE_REGION-regionserver/bd6592c37bb2:0-0 {event_type=M_RS_CLOSE_REGION, pid=55}] regionserver.HRegion(1676): Region close journal for 63842f73863e9e7b9587486b122e6aae: Waiting for close lock at 1732486486548Running coprocessor pre-close hooks at 1732486486548Disabling compacts and flushes for region at 1732486486548Disabling writes for close at 1732486486831 (+283 ms)Obtaining lock to block concurrent updates at 1732486486831Preparing flush snapshotting stores in 63842f73863e9e7b9587486b122e6aae at 1732486486831Finished memstore snapshotting TestAcidGuarantees,,1732486460633.63842f73863e9e7b9587486b122e6aae., syncing WAL and waiting on mvcc, flushsize=dataSize=27480, getHeapSize=82320, getOffHeapSize=0, getCellsCount=600 at 1732486486832 (+1 ms)Flushing stores of TestAcidGuarantees,,1732486460633.63842f73863e9e7b9587486b122e6aae. at 1732486486832Flushing 63842f73863e9e7b9587486b122e6aae/A: creating writer at 1732486486832Flushing mob file 63842f73863e9e7b9587486b122e6aae/A: appending metadata at 1732486486839 (+7 ms)Flushing mob file 63842f73863e9e7b9587486b122e6aae/A: closing flushed file at 1732486486839Flushing 63842f73863e9e7b9587486b122e6aae/A: appending metadata at 1732486487252 (+413 ms)Flushing 63842f73863e9e7b9587486b122e6aae/A: closing flushed file at 1732486487252Flushing 63842f73863e9e7b9587486b122e6aae/B: creating writer at 1732486487268 (+16 ms)Flushing 63842f73863e9e7b9587486b122e6aae/B: appending metadata at 1732486487271 (+3 ms)Flushing 63842f73863e9e7b9587486b122e6aae/B: closing flushed file at 1732486487271Flushing 63842f73863e9e7b9587486b122e6aae/C: creating writer at 1732486487684 (+413 ms)Flushing 63842f73863e9e7b9587486b122e6aae/C: appending metadata at 1732486487687 (+3 ms)Flushing 63842f73863e9e7b9587486b122e6aae/C: closing flushed file at 1732486487687Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@699fb17: reopening flushed file at 1732486488103 (+416 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@7041ef74: reopening flushed file at 1732486488107 (+4 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@125b1897: reopening flushed file at 1732486488111 (+4 ms)Finished flush of dataSize ~26.84 KB/27480, heapSize ~80.39 KB/82320, currentSize=0 B/0 for 63842f73863e9e7b9587486b122e6aae in 1284ms, sequenceid=494, compaction requested=false at 1732486488115 (+4 ms)Writing region close event to WAL at 1732486488209 (+94 ms)Running coprocessor post-close hooks at 1732486488213 (+4 ms)Closed at 1732486488213 2024-11-24T22:14:48,215 INFO [RS_CLOSE_REGION-regionserver/bd6592c37bb2:0-0 {event_type=M_RS_CLOSE_REGION, pid=55}] handler.UnassignRegionHandler(157): Closed 63842f73863e9e7b9587486b122e6aae 2024-11-24T22:14:48,216 INFO [PEWorker-5 {}] assignment.RegionStateStore(223): pid=54 updating hbase:meta row=63842f73863e9e7b9587486b122e6aae, regionState=CLOSED 2024-11-24T22:14:48,217 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=55, ppid=54, state=RUNNABLE, hasLock=false; CloseRegionProcedure 63842f73863e9e7b9587486b122e6aae, server=bd6592c37bb2,40087,1732486423713 because future has completed 2024-11-24T22:14:48,219 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=55, resume processing ppid=54 2024-11-24T22:14:48,219 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=55, ppid=54, state=SUCCESS, hasLock=false; CloseRegionProcedure 63842f73863e9e7b9587486b122e6aae, server=bd6592c37bb2,40087,1732486423713 in 1.8210 sec 2024-11-24T22:14:48,220 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=54, resume processing ppid=53 2024-11-24T22:14:48,220 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=54, ppid=53, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=63842f73863e9e7b9587486b122e6aae, UNASSIGN in 1.8270 sec 2024-11-24T22:14:48,221 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=53, resume processing ppid=52 2024-11-24T22:14:48,222 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=53, ppid=52, state=SUCCESS, hasLock=false; CloseTableRegionsProcedure table=TestAcidGuarantees in 1.8300 sec 2024-11-24T22:14:48,223 DEBUG [PEWorker-3 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1732486488223"}]},"ts":"1732486488223"} 2024-11-24T22:14:48,224 INFO [PEWorker-3 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=DISABLED in hbase:meta 2024-11-24T22:14:48,224 INFO [PEWorker-3 {}] procedure.DisableTableProcedure(296): Set TestAcidGuarantees to state=DISABLED 2024-11-24T22:14:48,225 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=52, state=SUCCESS, hasLock=false; DisableTableProcedure table=TestAcidGuarantees in 1.8390 sec 2024-11-24T22:14:48,531 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37701 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=52 2024-11-24T22:14:48,531 INFO [RPCClient-NioEventLoopGroup-4-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: DISABLE, Table Name: default:TestAcidGuarantees completed 2024-11-24T22:14:48,531 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37701 {}] master.HMaster$5(2570): Client=jenkins//172.17.0.2 delete TestAcidGuarantees 2024-11-24T22:14:48,533 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37701 {}] procedure2.ProcedureExecutor(1139): Stored pid=56, state=RUNNABLE:DELETE_TABLE_PRE_OPERATION, hasLock=false; DeleteTableProcedure table=TestAcidGuarantees 2024-11-24T22:14:48,534 DEBUG [PEWorker-5 {}] procedure.DeleteTableProcedure(103): Waiting for RIT for pid=56, state=RUNNABLE:DELETE_TABLE_PRE_OPERATION, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-24T22:14:48,534 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37701 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=56 2024-11-24T22:14:48,534 DEBUG [PEWorker-5 {}] procedure.DeleteTableProcedure(115): Deleting regions from filesystem for pid=56, state=RUNNABLE:DELETE_TABLE_CLEAR_FS_LAYOUT, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-24T22:14:48,537 DEBUG [HFileArchiver-2 {}] backup.HFileArchiver(131): ARCHIVING hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/63842f73863e9e7b9587486b122e6aae 2024-11-24T22:14:48,539 DEBUG [HFileArchiver-2 {}] backup.HFileArchiver(159): Archiving [FileablePath, hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/63842f73863e9e7b9587486b122e6aae/A, FileablePath, hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/63842f73863e9e7b9587486b122e6aae/B, FileablePath, hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/63842f73863e9e7b9587486b122e6aae/C, FileablePath, hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/63842f73863e9e7b9587486b122e6aae/recovered.edits] 2024-11-24T22:14:48,542 DEBUG [HFileArchiver-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/63842f73863e9e7b9587486b122e6aae/A/787c43c747aa44eea1b31a88c2ef6646 to hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/archive/data/default/TestAcidGuarantees/63842f73863e9e7b9587486b122e6aae/A/787c43c747aa44eea1b31a88c2ef6646 2024-11-24T22:14:48,543 DEBUG [HFileArchiver-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/63842f73863e9e7b9587486b122e6aae/A/8710268c8f6f4017b204d10b6ecccc64 to hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/archive/data/default/TestAcidGuarantees/63842f73863e9e7b9587486b122e6aae/A/8710268c8f6f4017b204d10b6ecccc64 2024-11-24T22:14:48,545 DEBUG [HFileArchiver-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/63842f73863e9e7b9587486b122e6aae/B/ad92a2ba5205468685ba83de56b0905a to hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/archive/data/default/TestAcidGuarantees/63842f73863e9e7b9587486b122e6aae/B/ad92a2ba5205468685ba83de56b0905a 2024-11-24T22:14:48,547 DEBUG [HFileArchiver-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/63842f73863e9e7b9587486b122e6aae/B/e6b46686d51a490199dd02551a78110d to hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/archive/data/default/TestAcidGuarantees/63842f73863e9e7b9587486b122e6aae/B/e6b46686d51a490199dd02551a78110d 2024-11-24T22:14:48,549 DEBUG [HFileArchiver-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/63842f73863e9e7b9587486b122e6aae/C/161093560a95433a9578b4c558493beb to hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/archive/data/default/TestAcidGuarantees/63842f73863e9e7b9587486b122e6aae/C/161093560a95433a9578b4c558493beb 2024-11-24T22:14:48,550 DEBUG [HFileArchiver-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/63842f73863e9e7b9587486b122e6aae/C/6db7eb2eee4a4a6b860ce55d7660e23d to hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/archive/data/default/TestAcidGuarantees/63842f73863e9e7b9587486b122e6aae/C/6db7eb2eee4a4a6b860ce55d7660e23d 2024-11-24T22:14:48,553 DEBUG [HFileArchiver-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/63842f73863e9e7b9587486b122e6aae/recovered.edits/497.seqid to hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/archive/data/default/TestAcidGuarantees/63842f73863e9e7b9587486b122e6aae/recovered.edits/497.seqid 2024-11-24T22:14:48,553 DEBUG [HFileArchiver-2 {}] backup.HFileArchiver(610): Deleted hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/63842f73863e9e7b9587486b122e6aae 2024-11-24T22:14:48,554 DEBUG [PEWorker-5 {}] procedure.DeleteTableProcedure(313): Archived TestAcidGuarantees regions 2024-11-24T22:14:48,554 DEBUG [PEWorker-5 {}] backup.HFileArchiver(131): ARCHIVING hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3 2024-11-24T22:14:48,555 DEBUG [PEWorker-5 {}] backup.HFileArchiver(159): Archiving [FileablePath, hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A] 2024-11-24T22:14:48,558 DEBUG [PEWorker-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024112400f4cd1cec9a4a14861e26f4dc88f79b_63842f73863e9e7b9587486b122e6aae to hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024112400f4cd1cec9a4a14861e26f4dc88f79b_63842f73863e9e7b9587486b122e6aae 2024-11-24T22:14:48,559 DEBUG [PEWorker-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411240dd7075793d44845b489eb8dcfe3af1c_63842f73863e9e7b9587486b122e6aae to hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411240dd7075793d44845b489eb8dcfe3af1c_63842f73863e9e7b9587486b122e6aae 2024-11-24T22:14:48,561 DEBUG [PEWorker-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411241184d2c472984a59a5ea93a8a3fb1025_63842f73863e9e7b9587486b122e6aae to hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411241184d2c472984a59a5ea93a8a3fb1025_63842f73863e9e7b9587486b122e6aae 2024-11-24T22:14:48,562 DEBUG [PEWorker-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024112426339100592e4032b4171064d82de25b_63842f73863e9e7b9587486b122e6aae to hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024112426339100592e4032b4171064d82de25b_63842f73863e9e7b9587486b122e6aae 2024-11-24T22:14:48,563 DEBUG [PEWorker-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024112428689bd4df0c47fc925b82ed3bc805e2_63842f73863e9e7b9587486b122e6aae to hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024112428689bd4df0c47fc925b82ed3bc805e2_63842f73863e9e7b9587486b122e6aae 2024-11-24T22:14:48,564 DEBUG [PEWorker-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411242c097a4926bf4978b6282666bacb85cc_63842f73863e9e7b9587486b122e6aae to hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411242c097a4926bf4978b6282666bacb85cc_63842f73863e9e7b9587486b122e6aae 2024-11-24T22:14:48,565 DEBUG [PEWorker-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411243ad647717a8d43cda58954ccaded35ca_63842f73863e9e7b9587486b122e6aae to hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411243ad647717a8d43cda58954ccaded35ca_63842f73863e9e7b9587486b122e6aae 2024-11-24T22:14:48,566 DEBUG [PEWorker-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411243b844e2044e54833a729059664d61a8b_63842f73863e9e7b9587486b122e6aae to hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411243b844e2044e54833a729059664d61a8b_63842f73863e9e7b9587486b122e6aae 2024-11-24T22:14:48,567 DEBUG [PEWorker-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411243b9b5f14f2294190802454150d5ad535_63842f73863e9e7b9587486b122e6aae to hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411243b9b5f14f2294190802454150d5ad535_63842f73863e9e7b9587486b122e6aae 2024-11-24T22:14:48,568 DEBUG [PEWorker-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411243f41dba08a4a449da306dd6125b8f2ac_63842f73863e9e7b9587486b122e6aae to hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411243f41dba08a4a449da306dd6125b8f2ac_63842f73863e9e7b9587486b122e6aae 2024-11-24T22:14:48,569 DEBUG [PEWorker-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411244231586579d04048854e6a51911a219c_63842f73863e9e7b9587486b122e6aae to hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411244231586579d04048854e6a51911a219c_63842f73863e9e7b9587486b122e6aae 2024-11-24T22:14:48,570 DEBUG [PEWorker-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024112466703d3eec03428db38bbc64c672f9c9_63842f73863e9e7b9587486b122e6aae to hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024112466703d3eec03428db38bbc64c672f9c9_63842f73863e9e7b9587486b122e6aae 2024-11-24T22:14:48,572 DEBUG [PEWorker-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411247610d45b6bd7401eb3c8ff9a83fd52f9_63842f73863e9e7b9587486b122e6aae to hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411247610d45b6bd7401eb3c8ff9a83fd52f9_63842f73863e9e7b9587486b122e6aae 2024-11-24T22:14:48,573 DEBUG [PEWorker-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024112488cadac551bc49149d727fb10c26b2cc_63842f73863e9e7b9587486b122e6aae to hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024112488cadac551bc49149d727fb10c26b2cc_63842f73863e9e7b9587486b122e6aae 2024-11-24T22:14:48,574 DEBUG [PEWorker-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241124903c41653ee343e4bdfa839688befd68_63842f73863e9e7b9587486b122e6aae to hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241124903c41653ee343e4bdfa839688befd68_63842f73863e9e7b9587486b122e6aae 2024-11-24T22:14:48,576 DEBUG [PEWorker-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411249831828caf7f4782b0bc8a54ad20c973_63842f73863e9e7b9587486b122e6aae to hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411249831828caf7f4782b0bc8a54ad20c973_63842f73863e9e7b9587486b122e6aae 2024-11-24T22:14:48,577 DEBUG [PEWorker-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241124a8603a69dd26464b9c54ecffa9c61a8c_63842f73863e9e7b9587486b122e6aae to hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241124a8603a69dd26464b9c54ecffa9c61a8c_63842f73863e9e7b9587486b122e6aae 2024-11-24T22:14:48,578 DEBUG [PEWorker-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241124a88f4a2c6d31436c856417b2bf385c03_63842f73863e9e7b9587486b122e6aae to hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241124a88f4a2c6d31436c856417b2bf385c03_63842f73863e9e7b9587486b122e6aae 2024-11-24T22:14:48,579 DEBUG [PEWorker-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241124c53264bab37e4ca68e6dc4d5f7177c66_63842f73863e9e7b9587486b122e6aae to hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241124c53264bab37e4ca68e6dc4d5f7177c66_63842f73863e9e7b9587486b122e6aae 2024-11-24T22:14:48,580 DEBUG [PEWorker-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241124dff701258fc247db9f301f1ffefc5958_63842f73863e9e7b9587486b122e6aae to hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241124dff701258fc247db9f301f1ffefc5958_63842f73863e9e7b9587486b122e6aae 2024-11-24T22:14:48,581 DEBUG [PEWorker-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241124e2c7467f191942dbb8139a28109001ad_63842f73863e9e7b9587486b122e6aae to hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241124e2c7467f191942dbb8139a28109001ad_63842f73863e9e7b9587486b122e6aae 2024-11-24T22:14:48,583 DEBUG [PEWorker-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241124fae42328bfa745698e2372ca001493d4_63842f73863e9e7b9587486b122e6aae to hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241124fae42328bfa745698e2372ca001493d4_63842f73863e9e7b9587486b122e6aae 2024-11-24T22:14:48,584 DEBUG [PEWorker-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241124fb2e73668a25415b989fcc090c3ce251_63842f73863e9e7b9587486b122e6aae to hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241124fb2e73668a25415b989fcc090c3ce251_63842f73863e9e7b9587486b122e6aae 2024-11-24T22:14:48,584 DEBUG [PEWorker-5 {}] backup.HFileArchiver(610): Deleted hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3 2024-11-24T22:14:48,586 DEBUG [PEWorker-5 {}] procedure.DeleteTableProcedure(120): Deleting regions from META for pid=56, state=RUNNABLE:DELETE_TABLE_REMOVE_FROM_META, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-24T22:14:48,589 WARN [PEWorker-5 {}] procedure.DeleteTableProcedure(370): Deleting some vestigial 1 rows of TestAcidGuarantees from hbase:meta 2024-11-24T22:14:48,591 DEBUG [PEWorker-5 {}] procedure.DeleteTableProcedure(407): Removing 'TestAcidGuarantees' descriptor. 2024-11-24T22:14:48,592 DEBUG [PEWorker-5 {}] procedure.DeleteTableProcedure(126): Deleting assignment state for pid=56, state=RUNNABLE:DELETE_TABLE_UNASSIGN_REGIONS, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-24T22:14:48,592 DEBUG [PEWorker-5 {}] procedure.DeleteTableProcedure(397): Removing 'TestAcidGuarantees' from region states. 2024-11-24T22:14:48,592 DEBUG [PEWorker-5 {}] assignment.RegionStateStore(723): Delete {"totalColumns":1,"row":"TestAcidGuarantees,,1732486460633.63842f73863e9e7b9587486b122e6aae.","families":{"info":[{"qualifier":"","vlen":0,"tag":[],"timestamp":"1732486488592"}]},"ts":"9223372036854775807"} 2024-11-24T22:14:48,594 INFO [PEWorker-5 {}] assignment.RegionStateStore(562): Deleted 1 regions from META 2024-11-24T22:14:48,594 DEBUG [PEWorker-5 {}] assignment.RegionStateStore(563): Deleted regions: [{ENCODED => 63842f73863e9e7b9587486b122e6aae, NAME => 'TestAcidGuarantees,,1732486460633.63842f73863e9e7b9587486b122e6aae.', STARTKEY => '', ENDKEY => ''}] 2024-11-24T22:14:48,594 DEBUG [PEWorker-5 {}] procedure.DeleteTableProcedure(401): Marking 'TestAcidGuarantees' as deleted. 2024-11-24T22:14:48,594 DEBUG [PEWorker-5 {}] hbase.MetaTableAccessor(964): Delete {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":0,"tag":[],"timestamp":"1732486488594"}]},"ts":"9223372036854775807"} 2024-11-24T22:14:48,596 INFO [PEWorker-5 {}] hbase.MetaTableAccessor(867): Deleted table TestAcidGuarantees state from META 2024-11-24T22:14:48,596 DEBUG [PEWorker-5 {}] procedure.DeleteTableProcedure(133): Finished pid=56, state=RUNNABLE:DELETE_TABLE_POST_OPERATION, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-24T22:14:48,597 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=56, state=SUCCESS, hasLock=false; DeleteTableProcedure table=TestAcidGuarantees in 64 msec 2024-11-24T22:14:48,641 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37701 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=56 2024-11-24T22:14:48,641 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocator(219): Clear meta cache for TestAcidGuarantees 2024-11-24T22:14:48,641 INFO [RPCClient-NioEventLoopGroup-4-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: DELETE, Table Name: default:TestAcidGuarantees completed 2024-11-24T22:14:48,658 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: TestAcidGuaranteesWithEagerPolicy#testMobMixedAtomicity Thread=253 (was 247) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_-954571297_22 at /127.0.0.1:47274 [Waiting for operation #743] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: ProcedureExecutor-Async-Task-Executor-4 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_-954571297_22 at /127.0.0.1:47266 [Waiting for operation #786] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_-1699160809_22 at /127.0.0.1:47314 [Waiting for operation #600] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_-1699160809_22 at /127.0.0.1:50438 [Waiting for operation #949] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: ProcedureExecutor-Async-Task-Executor-7 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: HFileArchiver-2 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: ProcedureExecutor-Async-Task-Executor-6 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: ProcedureExecutor-Async-Task-Executor-5 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: Async disk worker #0 for volume /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/c25542eb-68a7-66d5-8606-05e68b15f363/cluster_c8a06051-0388-dc48-080e-6657ef7cdb74/data/data1 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_-1699160809_22 at /127.0.0.1:44202 [Waiting for operation #957] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: Async disk worker #0 for volume /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/c25542eb-68a7-66d5-8606-05e68b15f363/cluster_c8a06051-0388-dc48-080e-6657ef7cdb74/data/data2 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) - Thread LEAK? -, OpenFileDescriptor=465 (was 445) - OpenFileDescriptor LEAK? -, MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=472 (was 347) - SystemLoadAverage LEAK? -, ProcessCount=11 (was 11), AvailableMemoryMB=6707 (was 7040) 2024-11-24T22:14:48,672 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: TestAcidGuaranteesWithEagerPolicy#testGetAtomicity Thread=253, OpenFileDescriptor=465, MaxFileDescriptor=1048576, SystemLoadAverage=472, ProcessCount=11, AvailableMemoryMB=6704 2024-11-24T22:14:48,674 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37701 {}] util.TableDescriptorChecker(321): MEMSTORE_FLUSHSIZE for table descriptor or "hbase.hregion.memstore.flush.size" (131072) is too small, which might cause very frequent flushing. 2024-11-24T22:14:48,675 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37701 {}] master.HMaster$4(2454): Client=jenkins//172.17.0.2 create 'TestAcidGuarantees', {TABLE_ATTRIBUTES => {METADATA => {'hbase.hregion.compacting.memstore.type' => 'EAGER', 'hbase.hregion.memstore.mslab.enabled' => 'false', 'hbase.memstore.inmemoryflush.threshold.factor' => '0.9'}}}, {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-11-24T22:14:48,676 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37701 {}] procedure2.ProcedureExecutor(1139): Stored pid=57, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=false; CreateTableProcedure table=TestAcidGuarantees 2024-11-24T22:14:48,677 INFO [PEWorker-4 {}] procedure.CreateTableProcedure(89): pid=57, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_PRE_OPERATION 2024-11-24T22:14:48,677 DEBUG [PEWorker-4 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T22:14:48,678 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37701 {}] master.MasterRpcServices(787): Client=jenkins//172.17.0.2 procedure request for creating table: namespace: "default" qualifier: "TestAcidGuarantees" procId is: 57 2024-11-24T22:14:48,678 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37701 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=57 2024-11-24T22:14:48,678 INFO [PEWorker-4 {}] procedure.CreateTableProcedure(89): pid=57, state=RUNNABLE:CREATE_TABLE_WRITE_FS_LAYOUT, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_WRITE_FS_LAYOUT 2024-11-24T22:14:48,684 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45929 is added to blk_1073742067_1243 (size=1061) 2024-11-24T22:14:48,780 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37701 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=57 2024-11-24T22:14:48,991 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37701 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=57 2024-11-24T22:14:49,087 INFO [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(7572): creating {ENCODED => cdb3383a4019da5f5ef0568dfacd542e, NAME => 'TestAcidGuarantees,,1732486488674.cdb3383a4019da5f5ef0568dfacd542e.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='TestAcidGuarantees', {TABLE_ATTRIBUTES => {METADATA => {'hbase.hregion.compacting.memstore.type' => 'EAGER', 'hbase.hregion.memstore.mslab.enabled' => 'false', 'hbase.memstore.inmemoryflush.threshold.factor' => '0.9', 'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc 2024-11-24T22:14:49,094 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45929 is added to blk_1073742068_1244 (size=53) 2024-11-24T22:14:49,302 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37701 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=57 2024-11-24T22:14:49,495 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(898): Instantiated TestAcidGuarantees,,1732486488674.cdb3383a4019da5f5ef0568dfacd542e.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-24T22:14:49,495 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1722): Closing cdb3383a4019da5f5ef0568dfacd542e, disabling compactions & flushes 2024-11-24T22:14:49,495 INFO [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1755): Closing region TestAcidGuarantees,,1732486488674.cdb3383a4019da5f5ef0568dfacd542e. 2024-11-24T22:14:49,495 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1776): Time limited wait for close lock on TestAcidGuarantees,,1732486488674.cdb3383a4019da5f5ef0568dfacd542e. 2024-11-24T22:14:49,495 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1843): Acquired close lock on TestAcidGuarantees,,1732486488674.cdb3383a4019da5f5ef0568dfacd542e. after waiting 0 ms 2024-11-24T22:14:49,495 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1853): Updates disabled for region TestAcidGuarantees,,1732486488674.cdb3383a4019da5f5ef0568dfacd542e. 2024-11-24T22:14:49,495 INFO [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1973): Closed TestAcidGuarantees,,1732486488674.cdb3383a4019da5f5ef0568dfacd542e. 2024-11-24T22:14:49,495 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1676): Region close journal for cdb3383a4019da5f5ef0568dfacd542e: Waiting for close lock at 1732486489495Disabling compacts and flushes for region at 1732486489495Disabling writes for close at 1732486489495Writing region close event to WAL at 1732486489495Closed at 1732486489495 2024-11-24T22:14:49,496 INFO [PEWorker-4 {}] procedure.CreateTableProcedure(89): pid=57, state=RUNNABLE:CREATE_TABLE_ADD_TO_META, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_ADD_TO_META 2024-11-24T22:14:49,496 DEBUG [PEWorker-4 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":2,"row":"TestAcidGuarantees,,1732486488674.cdb3383a4019da5f5ef0568dfacd542e.","families":{"info":[{"qualifier":"regioninfo","vlen":52,"tag":[],"timestamp":"1732486489496"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1732486489496"}]},"ts":"1732486489496"} 2024-11-24T22:14:49,498 INFO [PEWorker-4 {}] hbase.MetaTableAccessor(832): Added 1 regions to meta. 2024-11-24T22:14:49,499 INFO [PEWorker-4 {}] procedure.CreateTableProcedure(89): pid=57, state=RUNNABLE:CREATE_TABLE_ASSIGN_REGIONS, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_ASSIGN_REGIONS 2024-11-24T22:14:49,499 DEBUG [PEWorker-4 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1732486489499"}]},"ts":"1732486489499"} 2024-11-24T22:14:49,501 INFO [PEWorker-4 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=ENABLING in hbase:meta 2024-11-24T22:14:49,501 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=58, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=cdb3383a4019da5f5ef0568dfacd542e, ASSIGN}] 2024-11-24T22:14:49,502 INFO [PEWorker-2 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=58, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=cdb3383a4019da5f5ef0568dfacd542e, ASSIGN 2024-11-24T22:14:49,502 INFO [PEWorker-2 {}] assignment.TransitRegionStateProcedure(269): Starting pid=58, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=cdb3383a4019da5f5ef0568dfacd542e, ASSIGN; state=OFFLINE, location=bd6592c37bb2,40087,1732486423713; forceNewPlan=false, retain=false 2024-11-24T22:14:49,653 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=58 updating hbase:meta row=cdb3383a4019da5f5ef0568dfacd542e, regionState=OPENING, regionLocation=bd6592c37bb2,40087,1732486423713 2024-11-24T22:14:49,655 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=58, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=cdb3383a4019da5f5ef0568dfacd542e, ASSIGN because future has completed 2024-11-24T22:14:49,655 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=59, ppid=58, state=RUNNABLE, hasLock=false; OpenRegionProcedure cdb3383a4019da5f5ef0568dfacd542e, server=bd6592c37bb2,40087,1732486423713}] 2024-11-24T22:14:49,810 INFO [RS_OPEN_REGION-regionserver/bd6592c37bb2:0-0 {event_type=M_RS_OPEN_REGION, pid=59}] handler.AssignRegionHandler(132): Open TestAcidGuarantees,,1732486488674.cdb3383a4019da5f5ef0568dfacd542e. 2024-11-24T22:14:49,811 DEBUG [RS_OPEN_REGION-regionserver/bd6592c37bb2:0-0 {event_type=M_RS_OPEN_REGION, pid=59}] regionserver.HRegion(7752): Opening region: {ENCODED => cdb3383a4019da5f5ef0568dfacd542e, NAME => 'TestAcidGuarantees,,1732486488674.cdb3383a4019da5f5ef0568dfacd542e.', STARTKEY => '', ENDKEY => ''} 2024-11-24T22:14:49,811 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37701 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=57 2024-11-24T22:14:49,811 DEBUG [RS_OPEN_REGION-regionserver/bd6592c37bb2:0-0 {event_type=M_RS_OPEN_REGION, pid=59}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table TestAcidGuarantees cdb3383a4019da5f5ef0568dfacd542e 2024-11-24T22:14:49,811 DEBUG [RS_OPEN_REGION-regionserver/bd6592c37bb2:0-0 {event_type=M_RS_OPEN_REGION, pid=59}] regionserver.HRegion(898): Instantiated TestAcidGuarantees,,1732486488674.cdb3383a4019da5f5ef0568dfacd542e.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-24T22:14:49,811 DEBUG [RS_OPEN_REGION-regionserver/bd6592c37bb2:0-0 {event_type=M_RS_OPEN_REGION, pid=59}] regionserver.HRegion(7794): checking encryption for cdb3383a4019da5f5ef0568dfacd542e 2024-11-24T22:14:49,811 DEBUG [RS_OPEN_REGION-regionserver/bd6592c37bb2:0-0 {event_type=M_RS_OPEN_REGION, pid=59}] regionserver.HRegion(7797): checking classloading for cdb3383a4019da5f5ef0568dfacd542e 2024-11-24T22:14:49,813 INFO [StoreOpener-cdb3383a4019da5f5ef0568dfacd542e-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family A of region cdb3383a4019da5f5ef0568dfacd542e 2024-11-24T22:14:49,814 INFO [StoreOpener-cdb3383a4019da5f5ef0568dfacd542e-1 {}] regionserver.CompactingMemStore(122): Store=A, in-memory flush size threshold=38.40 KB, immutable segments index type=ARRAY_MAP, compactor=EAGER, pipelineThreshold=2, compactionCellMax=10 2024-11-24T22:14:49,814 INFO [StoreOpener-cdb3383a4019da5f5ef0568dfacd542e-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region cdb3383a4019da5f5ef0568dfacd542e columnFamilyName A 2024-11-24T22:14:49,814 DEBUG [StoreOpener-cdb3383a4019da5f5ef0568dfacd542e-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T22:14:49,815 INFO [StoreOpener-cdb3383a4019da5f5ef0568dfacd542e-1 {}] regionserver.HStore(327): Store=cdb3383a4019da5f5ef0568dfacd542e/A, memstore type=CompactingMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-24T22:14:49,815 INFO [StoreOpener-cdb3383a4019da5f5ef0568dfacd542e-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family B of region cdb3383a4019da5f5ef0568dfacd542e 2024-11-24T22:14:49,816 INFO [StoreOpener-cdb3383a4019da5f5ef0568dfacd542e-1 {}] regionserver.CompactingMemStore(122): Store=B, in-memory flush size threshold=38.40 KB, immutable segments index type=ARRAY_MAP, compactor=EAGER, pipelineThreshold=2, compactionCellMax=10 2024-11-24T22:14:49,817 INFO [StoreOpener-cdb3383a4019da5f5ef0568dfacd542e-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region cdb3383a4019da5f5ef0568dfacd542e columnFamilyName B 2024-11-24T22:14:49,817 DEBUG [StoreOpener-cdb3383a4019da5f5ef0568dfacd542e-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T22:14:49,817 INFO [StoreOpener-cdb3383a4019da5f5ef0568dfacd542e-1 {}] regionserver.HStore(327): Store=cdb3383a4019da5f5ef0568dfacd542e/B, memstore type=CompactingMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-24T22:14:49,817 INFO [StoreOpener-cdb3383a4019da5f5ef0568dfacd542e-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family C of region cdb3383a4019da5f5ef0568dfacd542e 2024-11-24T22:14:49,818 INFO [StoreOpener-cdb3383a4019da5f5ef0568dfacd542e-1 {}] regionserver.CompactingMemStore(122): Store=C, in-memory flush size threshold=38.40 KB, immutable segments index type=ARRAY_MAP, compactor=EAGER, pipelineThreshold=2, compactionCellMax=10 2024-11-24T22:14:49,818 INFO [StoreOpener-cdb3383a4019da5f5ef0568dfacd542e-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region cdb3383a4019da5f5ef0568dfacd542e columnFamilyName C 2024-11-24T22:14:49,818 DEBUG [StoreOpener-cdb3383a4019da5f5ef0568dfacd542e-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T22:14:49,819 INFO [StoreOpener-cdb3383a4019da5f5ef0568dfacd542e-1 {}] regionserver.HStore(327): Store=cdb3383a4019da5f5ef0568dfacd542e/C, memstore type=CompactingMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-24T22:14:49,819 INFO [RS_OPEN_REGION-regionserver/bd6592c37bb2:0-0 {event_type=M_RS_OPEN_REGION, pid=59}] regionserver.HRegion(1212): Setting FlushNonSloppyStoresFirstPolicy for the region=TestAcidGuarantees,,1732486488674.cdb3383a4019da5f5ef0568dfacd542e. 2024-11-24T22:14:49,819 DEBUG [RS_OPEN_REGION-regionserver/bd6592c37bb2:0-0 {event_type=M_RS_OPEN_REGION, pid=59}] regionserver.HRegion(1038): replaying wal for cdb3383a4019da5f5ef0568dfacd542e 2024-11-24T22:14:49,820 DEBUG [RS_OPEN_REGION-regionserver/bd6592c37bb2:0-0 {event_type=M_RS_OPEN_REGION, pid=59}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/cdb3383a4019da5f5ef0568dfacd542e 2024-11-24T22:14:49,820 DEBUG [RS_OPEN_REGION-regionserver/bd6592c37bb2:0-0 {event_type=M_RS_OPEN_REGION, pid=59}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/cdb3383a4019da5f5ef0568dfacd542e 2024-11-24T22:14:49,821 DEBUG [RS_OPEN_REGION-regionserver/bd6592c37bb2:0-0 {event_type=M_RS_OPEN_REGION, pid=59}] regionserver.HRegion(1048): stopping wal replay for cdb3383a4019da5f5ef0568dfacd542e 2024-11-24T22:14:49,821 DEBUG [RS_OPEN_REGION-regionserver/bd6592c37bb2:0-0 {event_type=M_RS_OPEN_REGION, pid=59}] regionserver.HRegion(1060): Cleaning up temporary data for cdb3383a4019da5f5ef0568dfacd542e 2024-11-24T22:14:49,822 DEBUG [RS_OPEN_REGION-regionserver/bd6592c37bb2:0-0 {event_type=M_RS_OPEN_REGION, pid=59}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table TestAcidGuarantees descriptor;using region.getMemStoreFlushHeapSize/# of families (16.0 M)) instead. 2024-11-24T22:14:49,823 DEBUG [RS_OPEN_REGION-regionserver/bd6592c37bb2:0-0 {event_type=M_RS_OPEN_REGION, pid=59}] regionserver.HRegion(1093): writing seq id for cdb3383a4019da5f5ef0568dfacd542e 2024-11-24T22:14:49,825 DEBUG [RS_OPEN_REGION-regionserver/bd6592c37bb2:0-0 {event_type=M_RS_OPEN_REGION, pid=59}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/cdb3383a4019da5f5ef0568dfacd542e/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-24T22:14:49,826 INFO [RS_OPEN_REGION-regionserver/bd6592c37bb2:0-0 {event_type=M_RS_OPEN_REGION, pid=59}] regionserver.HRegion(1114): Opened cdb3383a4019da5f5ef0568dfacd542e; next sequenceid=2; ConstantSizeRegionSplitPolicy{desiredMaxFileSize=68644532, jitterRate=0.022883236408233643}, FlushLargeStoresPolicy{flushSizeLowerBound=16777216} 2024-11-24T22:14:49,826 DEBUG [RS_OPEN_REGION-regionserver/bd6592c37bb2:0-0 {event_type=M_RS_OPEN_REGION, pid=59}] regionserver.HRegion(1122): Running coprocessor post-open hooks for cdb3383a4019da5f5ef0568dfacd542e 2024-11-24T22:14:49,826 DEBUG [RS_OPEN_REGION-regionserver/bd6592c37bb2:0-0 {event_type=M_RS_OPEN_REGION, pid=59}] regionserver.HRegion(1006): Region open journal for cdb3383a4019da5f5ef0568dfacd542e: Running coprocessor pre-open hook at 1732486489811Writing region info on filesystem at 1732486489811Initializing all the Stores at 1732486489812 (+1 ms)Instantiating store for column family {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732486489812Instantiating store for column family {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732486489812Instantiating store for column family {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732486489813 (+1 ms)Cleaning up temporary data from old regions at 1732486489821 (+8 ms)Running coprocessor post-open hooks at 1732486489826 (+5 ms)Region opened successfully at 1732486489826 2024-11-24T22:14:49,827 INFO [RS_OPEN_REGION-regionserver/bd6592c37bb2:0-0 {event_type=M_RS_OPEN_REGION, pid=59}] regionserver.HRegionServer(2236): Post open deploy tasks for TestAcidGuarantees,,1732486488674.cdb3383a4019da5f5ef0568dfacd542e., pid=59, masterSystemTime=1732486489807 2024-11-24T22:14:49,829 DEBUG [RS_OPEN_REGION-regionserver/bd6592c37bb2:0-0 {event_type=M_RS_OPEN_REGION, pid=59}] regionserver.HRegionServer(2266): Finished post open deploy task for TestAcidGuarantees,,1732486488674.cdb3383a4019da5f5ef0568dfacd542e. 2024-11-24T22:14:49,829 INFO [RS_OPEN_REGION-regionserver/bd6592c37bb2:0-0 {event_type=M_RS_OPEN_REGION, pid=59}] handler.AssignRegionHandler(153): Opened TestAcidGuarantees,,1732486488674.cdb3383a4019da5f5ef0568dfacd542e. 2024-11-24T22:14:49,830 INFO [PEWorker-4 {}] assignment.RegionStateStore(223): pid=58 updating hbase:meta row=cdb3383a4019da5f5ef0568dfacd542e, regionState=OPEN, openSeqNum=2, regionLocation=bd6592c37bb2,40087,1732486423713 2024-11-24T22:14:49,831 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=59, ppid=58, state=RUNNABLE, hasLock=false; OpenRegionProcedure cdb3383a4019da5f5ef0568dfacd542e, server=bd6592c37bb2,40087,1732486423713 because future has completed 2024-11-24T22:14:49,834 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=59, resume processing ppid=58 2024-11-24T22:14:49,834 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=59, ppid=58, state=SUCCESS, hasLock=false; OpenRegionProcedure cdb3383a4019da5f5ef0568dfacd542e, server=bd6592c37bb2,40087,1732486423713 in 177 msec 2024-11-24T22:14:49,836 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=58, resume processing ppid=57 2024-11-24T22:14:49,836 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=58, ppid=57, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=cdb3383a4019da5f5ef0568dfacd542e, ASSIGN in 333 msec 2024-11-24T22:14:49,837 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=57, state=RUNNABLE:CREATE_TABLE_UPDATE_DESC_CACHE, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_UPDATE_DESC_CACHE 2024-11-24T22:14:49,838 DEBUG [PEWorker-3 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1732486489837"}]},"ts":"1732486489837"} 2024-11-24T22:14:49,839 INFO [PEWorker-3 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=ENABLED in hbase:meta 2024-11-24T22:14:49,840 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=57, state=RUNNABLE:CREATE_TABLE_POST_OPERATION, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_POST_OPERATION 2024-11-24T22:14:49,841 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=57, state=SUCCESS, hasLock=false; CreateTableProcedure table=TestAcidGuarantees in 1.1650 sec 2024-11-24T22:14:50,820 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37701 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=57 2024-11-24T22:14:50,821 INFO [RPCClient-NioEventLoopGroup-4-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: CREATE, Table Name: default:TestAcidGuarantees completed 2024-11-24T22:14:50,822 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@e04d2b7, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-24T22:14:50,822 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request bd6592c37bb2,37701,-1 for getting cluster id 2024-11-24T22:14:50,822 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-24T22:14:50,823 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '3c461ac1-06fb-4739-8702-6368e685b03c' 2024-11-24T22:14:50,823 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-24T22:14:50,823 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "3c461ac1-06fb-4739-8702-6368e685b03c" 2024-11-24T22:14:50,823 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@55c32776, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-24T22:14:50,823 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [bd6592c37bb2,37701,-1] 2024-11-24T22:14:50,823 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-24T22:14:50,823 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-24T22:14:50,824 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:36998, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-24T22:14:50,825 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@7b296778, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-24T22:14:50,825 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-24T22:14:50,826 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=bd6592c37bb2,40087,1732486423713, seqNum=-1] 2024-11-24T22:14:50,826 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-24T22:14:50,826 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:40742, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-24T22:14:50,827 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching master stub from registry 2024-11-24T22:14:50,828 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncConnectionImpl(321): The fetched master address is bd6592c37bb2,37701,1732486422992 2024-11-24T22:14:50,828 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.ConnectionUtils(555): The fetched master stub is org.apache.hadoop.hbase.shaded.protobuf.generated.MasterProtos$MasterService$Stub@b76f413 2024-11-24T22:14:50,828 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=MasterService, sasl=false 2024-11-24T22:14:50,829 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:37006, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=MasterService 2024-11-24T22:14:50,830 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@391cff66, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-24T22:14:50,830 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request bd6592c37bb2,37701,-1 for getting cluster id 2024-11-24T22:14:50,830 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-24T22:14:50,830 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '3c461ac1-06fb-4739-8702-6368e685b03c' 2024-11-24T22:14:50,830 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-24T22:14:50,830 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "3c461ac1-06fb-4739-8702-6368e685b03c" 2024-11-24T22:14:50,831 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@85303a, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-24T22:14:50,831 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [bd6592c37bb2,37701,-1] 2024-11-24T22:14:50,831 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-24T22:14:50,831 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-24T22:14:50,831 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:37018, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-24T22:14:50,832 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@63e55a92, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-24T22:14:50,832 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@135bd0ea, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-24T22:14:50,832 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request bd6592c37bb2,37701,-1 for getting cluster id 2024-11-24T22:14:50,833 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-24T22:14:50,833 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '3c461ac1-06fb-4739-8702-6368e685b03c' 2024-11-24T22:14:50,833 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-24T22:14:50,833 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "3c461ac1-06fb-4739-8702-6368e685b03c" 2024-11-24T22:14:50,833 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@58f38f67, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-24T22:14:50,833 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [bd6592c37bb2,37701,-1] 2024-11-24T22:14:50,833 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-24T22:14:50,834 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-24T22:14:50,834 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:37042, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-24T22:14:50,835 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@1f60e158, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-24T22:14:50,835 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@4609cead, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-24T22:14:50,835 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request bd6592c37bb2,37701,-1 for getting cluster id 2024-11-24T22:14:50,835 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-24T22:14:50,836 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '3c461ac1-06fb-4739-8702-6368e685b03c' 2024-11-24T22:14:50,836 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-24T22:14:50,836 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "3c461ac1-06fb-4739-8702-6368e685b03c" 2024-11-24T22:14:50,836 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@f080d47, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-24T22:14:50,836 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [bd6592c37bb2,37701,-1] 2024-11-24T22:14:50,836 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-24T22:14:50,836 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-24T22:14:50,837 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:37064, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-24T22:14:50,837 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@49710153, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-24T22:14:50,838 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@3a2df097, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-24T22:14:50,838 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request bd6592c37bb2,37701,-1 for getting cluster id 2024-11-24T22:14:50,838 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-24T22:14:50,838 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '3c461ac1-06fb-4739-8702-6368e685b03c' 2024-11-24T22:14:50,838 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-24T22:14:50,839 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "3c461ac1-06fb-4739-8702-6368e685b03c" 2024-11-24T22:14:50,839 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@703fbb65, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-24T22:14:50,839 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [bd6592c37bb2,37701,-1] 2024-11-24T22:14:50,839 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-24T22:14:50,839 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-24T22:14:50,839 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:37076, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-24T22:14:50,840 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@75f66c8c, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-24T22:14:50,840 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@2856abc4, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-24T22:14:50,840 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request bd6592c37bb2,37701,-1 for getting cluster id 2024-11-24T22:14:50,841 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-24T22:14:50,841 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '3c461ac1-06fb-4739-8702-6368e685b03c' 2024-11-24T22:14:50,841 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-24T22:14:50,841 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "3c461ac1-06fb-4739-8702-6368e685b03c" 2024-11-24T22:14:50,841 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@3368c779, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-24T22:14:50,841 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [bd6592c37bb2,37701,-1] 2024-11-24T22:14:50,842 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-24T22:14:50,842 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-24T22:14:50,842 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:37098, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-24T22:14:50,843 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@42d33b79, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-24T22:14:50,843 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@7a4f6020, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-24T22:14:50,843 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request bd6592c37bb2,37701,-1 for getting cluster id 2024-11-24T22:14:50,843 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-24T22:14:50,844 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '3c461ac1-06fb-4739-8702-6368e685b03c' 2024-11-24T22:14:50,844 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-24T22:14:50,844 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "3c461ac1-06fb-4739-8702-6368e685b03c" 2024-11-24T22:14:50,844 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@67c13161, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-24T22:14:50,844 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [bd6592c37bb2,37701,-1] 2024-11-24T22:14:50,844 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-24T22:14:50,844 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-24T22:14:50,845 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:37126, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-24T22:14:50,845 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@656c29c3, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-24T22:14:50,846 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@692cba37, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-24T22:14:50,846 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request bd6592c37bb2,37701,-1 for getting cluster id 2024-11-24T22:14:50,846 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-24T22:14:50,846 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '3c461ac1-06fb-4739-8702-6368e685b03c' 2024-11-24T22:14:50,847 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-24T22:14:50,847 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "3c461ac1-06fb-4739-8702-6368e685b03c" 2024-11-24T22:14:50,847 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@3fac94a5, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-24T22:14:50,847 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [bd6592c37bb2,37701,-1] 2024-11-24T22:14:50,847 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-24T22:14:50,847 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-24T22:14:50,848 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:37152, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-24T22:14:50,849 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@653ed6ad, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-24T22:14:50,850 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@1d805c88, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-24T22:14:50,850 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request bd6592c37bb2,37701,-1 for getting cluster id 2024-11-24T22:14:50,850 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-24T22:14:50,850 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '3c461ac1-06fb-4739-8702-6368e685b03c' 2024-11-24T22:14:50,850 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-24T22:14:50,851 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "3c461ac1-06fb-4739-8702-6368e685b03c" 2024-11-24T22:14:50,851 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@422a8fb8, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-24T22:14:50,851 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [bd6592c37bb2,37701,-1] 2024-11-24T22:14:50,851 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-24T22:14:50,851 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-24T22:14:50,852 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:37160, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-24T22:14:50,852 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@2a078dc3, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-24T22:14:50,852 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@4861c2d6, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-24T22:14:50,852 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request bd6592c37bb2,37701,-1 for getting cluster id 2024-11-24T22:14:50,853 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-24T22:14:50,853 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '3c461ac1-06fb-4739-8702-6368e685b03c' 2024-11-24T22:14:50,853 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-24T22:14:50,853 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "3c461ac1-06fb-4739-8702-6368e685b03c" 2024-11-24T22:14:50,853 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@3823cc8, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-24T22:14:50,853 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [bd6592c37bb2,37701,-1] 2024-11-24T22:14:50,853 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-24T22:14:50,854 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-24T22:14:50,854 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:37192, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-24T22:14:50,854 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@7aa6271d, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-24T22:14:50,855 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@3397c776, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-24T22:14:50,855 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request bd6592c37bb2,37701,-1 for getting cluster id 2024-11-24T22:14:50,855 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-24T22:14:50,856 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '3c461ac1-06fb-4739-8702-6368e685b03c' 2024-11-24T22:14:50,856 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-24T22:14:50,856 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "3c461ac1-06fb-4739-8702-6368e685b03c" 2024-11-24T22:14:50,856 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@26186581, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-24T22:14:50,856 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [bd6592c37bb2,37701,-1] 2024-11-24T22:14:50,856 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-24T22:14:50,856 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-24T22:14:50,857 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:37210, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-24T22:14:50,857 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@17d02d33, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-24T22:14:50,858 DEBUG [Thread-1143 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-24T22:14:50,858 DEBUG [Thread-1154 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-24T22:14:50,858 DEBUG [Thread-1148 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-24T22:14:50,858 DEBUG [Thread-1146 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-24T22:14:50,858 DEBUG [Thread-1152 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-24T22:14:50,858 DEBUG [Thread-1150 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-24T22:14:50,858 DEBUG [Thread-1137 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-24T22:14:50,858 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37701 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-24T22:14:50,858 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=bd6592c37bb2,40087,1732486423713, seqNum=-1] 2024-11-24T22:14:50,859 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=bd6592c37bb2,40087,1732486423713, seqNum=-1] 2024-11-24T22:14:50,859 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-24T22:14:50,859 DEBUG [Thread-1135 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-24T22:14:50,859 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-24T22:14:50,859 DEBUG [Thread-1141 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-24T22:14:50,859 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=bd6592c37bb2,40087,1732486423713, seqNum=-1] 2024-11-24T22:14:50,859 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=bd6592c37bb2,40087,1732486423713, seqNum=-1] 2024-11-24T22:14:50,859 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-24T22:14:50,860 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-24T22:14:50,860 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:40756, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-24T22:14:50,860 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=bd6592c37bb2,40087,1732486423713, seqNum=-1] 2024-11-24T22:14:50,860 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=bd6592c37bb2,40087,1732486423713, seqNum=-1] 2024-11-24T22:14:50,860 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-24T22:14:50,860 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-24T22:14:50,860 DEBUG [Thread-1139 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-24T22:14:50,860 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:40772, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-24T22:14:50,861 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:40782, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-24T22:14:50,861 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:40796, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-24T22:14:50,861 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=bd6592c37bb2,40087,1732486423713, seqNum=-1] 2024-11-24T22:14:50,861 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37701 {}] procedure2.ProcedureExecutor(1139): Stored pid=60, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=60, table=TestAcidGuarantees 2024-11-24T22:14:50,861 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-24T22:14:50,861 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=bd6592c37bb2,40087,1732486423713, seqNum=-1] 2024-11-24T22:14:50,861 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-24T22:14:50,862 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=bd6592c37bb2,40087,1732486423713, seqNum=-1] 2024-11-24T22:14:50,862 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-24T22:14:50,862 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37701 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=60 2024-11-24T22:14:50,862 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:40748, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-24T22:14:50,862 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:40806, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-24T22:14:50,878 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:40816, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-24T22:14:50,878 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_0', locateType=CURRENT is [region=TestAcidGuarantees,,1732486488674.cdb3383a4019da5f5ef0568dfacd542e., hostname=bd6592c37bb2,40087,1732486423713, seqNum=2] 2024-11-24T22:14:50,878 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:40818, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-24T22:14:50,878 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_2', locateType=CURRENT is [region=TestAcidGuarantees,,1732486488674.cdb3383a4019da5f5ef0568dfacd542e., hostname=bd6592c37bb2,40087,1732486423713, seqNum=2] 2024-11-24T22:14:50,879 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:40834, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-24T22:14:50,879 DEBUG [Thread-1139 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=bd6592c37bb2,40087,1732486423713, seqNum=-1] 2024-11-24T22:14:50,879 DEBUG [Thread-1139 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-24T22:14:50,880 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_0', locateType=CURRENT is [region=TestAcidGuarantees,,1732486488674.cdb3383a4019da5f5ef0568dfacd542e., hostname=bd6592c37bb2,40087,1732486423713, seqNum=2] 2024-11-24T22:14:50,880 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=60, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=60, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-24T22:14:50,880 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_0', locateType=CURRENT is [region=TestAcidGuarantees,,1732486488674.cdb3383a4019da5f5ef0568dfacd542e., hostname=bd6592c37bb2,40087,1732486423713, seqNum=2] 2024-11-24T22:14:50,881 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_1', locateType=CURRENT is [region=TestAcidGuarantees,,1732486488674.cdb3383a4019da5f5ef0568dfacd542e., hostname=bd6592c37bb2,40087,1732486423713, seqNum=2] 2024-11-24T22:14:50,881 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_2', locateType=CURRENT is [region=TestAcidGuarantees,,1732486488674.cdb3383a4019da5f5ef0568dfacd542e., hostname=bd6592c37bb2,40087,1732486423713, seqNum=2] 2024-11-24T22:14:50,881 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=60, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=60, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-24T22:14:50,881 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=61, ppid=60, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-24T22:14:50,882 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:40842, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-24T22:14:50,882 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_2', locateType=CURRENT is [region=TestAcidGuarantees,,1732486488674.cdb3383a4019da5f5ef0568dfacd542e., hostname=bd6592c37bb2,40087,1732486423713, seqNum=2] 2024-11-24T22:14:50,882 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_1', locateType=CURRENT is [region=TestAcidGuarantees,,1732486488674.cdb3383a4019da5f5ef0568dfacd542e., hostname=bd6592c37bb2,40087,1732486423713, seqNum=2] 2024-11-24T22:14:50,883 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_1', locateType=CURRENT is [region=TestAcidGuarantees,,1732486488674.cdb3383a4019da5f5ef0568dfacd542e., hostname=bd6592c37bb2,40087,1732486423713, seqNum=2] 2024-11-24T22:14:50,883 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_2', locateType=CURRENT is [region=TestAcidGuarantees,,1732486488674.cdb3383a4019da5f5ef0568dfacd542e., hostname=bd6592c37bb2,40087,1732486423713, seqNum=2] 2024-11-24T22:14:50,889 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40087 {}] regionserver.HRegion(8855): Flush requested on cdb3383a4019da5f5ef0568dfacd542e 2024-11-24T22:14:50,889 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing cdb3383a4019da5f5ef0568dfacd542e 3/3 column families, dataSize=46.96 KB heapSize=140.20 KB 2024-11-24T22:14:50,890 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK cdb3383a4019da5f5ef0568dfacd542e, store=A 2024-11-24T22:14:50,890 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-24T22:14:50,890 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK cdb3383a4019da5f5ef0568dfacd542e, store=B 2024-11-24T22:14:50,890 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-24T22:14:50,890 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK cdb3383a4019da5f5ef0568dfacd542e, store=C 2024-11-24T22:14:50,890 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-24T22:14:50,925 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/cdb3383a4019da5f5ef0568dfacd542e/.tmp/A/4eb6e5cbfdd146468d5f9e4bee6f4417 is 50, key is test_row_0/A:col10/1732486490888/Put/seqid=0 2024-11-24T22:14:50,925 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/bd6592c37bb2:0-0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1732486490916, max timestamp=1732486490922Num uniques 859; 2024-11-24T22:14:50,925 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40087 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cdb3383a4019da5f5ef0568dfacd542e, server=bd6592c37bb2,40087,1732486423713 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-24T22:14:50,925 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/bd6592c37bb2:0-0 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-11-24T22:14:50,925 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40087 {}] ipc.CallRunner(138): callId: 5 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40834 deadline: 1732486550918, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cdb3383a4019da5f5ef0568dfacd542e, server=bd6592c37bb2,40087,1732486423713 2024-11-24T22:14:50,926 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732486488674.cdb3383a4019da5f5ef0568dfacd542e., hostname=bd6592c37bb2,40087,1732486423713, seqNum=2 , the old value is region=TestAcidGuarantees,,1732486488674.cdb3383a4019da5f5ef0568dfacd542e., hostname=bd6592c37bb2,40087,1732486423713, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cdb3383a4019da5f5ef0568dfacd542e, server=bd6592c37bb2,40087,1732486423713 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T22:14:50,926 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732486488674.cdb3383a4019da5f5ef0568dfacd542e., hostname=bd6592c37bb2,40087,1732486423713, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cdb3383a4019da5f5ef0568dfacd542e, server=bd6592c37bb2,40087,1732486423713 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T22:14:50,926 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732486488674.cdb3383a4019da5f5ef0568dfacd542e., hostname=bd6592c37bb2,40087,1732486423713, seqNum=2 because the exception is null or not the one we care about 2024-11-24T22:14:50,928 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/bd6592c37bb2:0-5 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1732486490916, max timestamp=1732486490922Num uniques 859; 2024-11-24T22:14:50,928 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/bd6592c37bb2:0-5 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-11-24T22:14:50,928 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/bd6592c37bb2:0-0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1732486490916, max timestamp=1732486490922Num uniques 859; 2024-11-24T22:14:50,928 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/bd6592c37bb2:0-0 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-11-24T22:14:50,943 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45929 is added to blk_1073742069_1245 (size=12001) 2024-11-24T22:14:50,944 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=12 (bloomFilter=true), to=hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/cdb3383a4019da5f5ef0568dfacd542e/.tmp/A/4eb6e5cbfdd146468d5f9e4bee6f4417 2024-11-24T22:14:50,963 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40087 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cdb3383a4019da5f5ef0568dfacd542e, server=bd6592c37bb2,40087,1732486423713 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-24T22:14:50,964 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40087 {}] ipc.CallRunner(138): callId: 11 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40772 deadline: 1732486550956, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cdb3383a4019da5f5ef0568dfacd542e, server=bd6592c37bb2,40087,1732486423713 2024-11-24T22:14:50,964 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40087 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cdb3383a4019da5f5ef0568dfacd542e, server=bd6592c37bb2,40087,1732486423713 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-24T22:14:50,964 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732486488674.cdb3383a4019da5f5ef0568dfacd542e., hostname=bd6592c37bb2,40087,1732486423713, seqNum=2 , the old value is region=TestAcidGuarantees,,1732486488674.cdb3383a4019da5f5ef0568dfacd542e., hostname=bd6592c37bb2,40087,1732486423713, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cdb3383a4019da5f5ef0568dfacd542e, server=bd6592c37bb2,40087,1732486423713 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T22:14:50,964 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40087 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40842 deadline: 1732486550958, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cdb3383a4019da5f5ef0568dfacd542e, server=bd6592c37bb2,40087,1732486423713 2024-11-24T22:14:50,964 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732486488674.cdb3383a4019da5f5ef0568dfacd542e., hostname=bd6592c37bb2,40087,1732486423713, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cdb3383a4019da5f5ef0568dfacd542e, server=bd6592c37bb2,40087,1732486423713 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T22:14:50,964 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732486488674.cdb3383a4019da5f5ef0568dfacd542e., hostname=bd6592c37bb2,40087,1732486423713, seqNum=2 because the exception is null or not the one we care about 2024-11-24T22:14:50,964 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732486488674.cdb3383a4019da5f5ef0568dfacd542e., hostname=bd6592c37bb2,40087,1732486423713, seqNum=2 , the old value is region=TestAcidGuarantees,,1732486488674.cdb3383a4019da5f5ef0568dfacd542e., hostname=bd6592c37bb2,40087,1732486423713, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cdb3383a4019da5f5ef0568dfacd542e, server=bd6592c37bb2,40087,1732486423713 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T22:14:50,965 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732486488674.cdb3383a4019da5f5ef0568dfacd542e., hostname=bd6592c37bb2,40087,1732486423713, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cdb3383a4019da5f5ef0568dfacd542e, server=bd6592c37bb2,40087,1732486423713 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T22:14:50,965 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732486488674.cdb3383a4019da5f5ef0568dfacd542e., hostname=bd6592c37bb2,40087,1732486423713, seqNum=2 because the exception is null or not the one we care about 2024-11-24T22:14:50,965 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40087 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cdb3383a4019da5f5ef0568dfacd542e, server=bd6592c37bb2,40087,1732486423713 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-24T22:14:50,965 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40087 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40816 deadline: 1732486550959, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cdb3383a4019da5f5ef0568dfacd542e, server=bd6592c37bb2,40087,1732486423713 2024-11-24T22:14:50,965 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732486488674.cdb3383a4019da5f5ef0568dfacd542e., hostname=bd6592c37bb2,40087,1732486423713, seqNum=2 , the old value is region=TestAcidGuarantees,,1732486488674.cdb3383a4019da5f5ef0568dfacd542e., hostname=bd6592c37bb2,40087,1732486423713, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cdb3383a4019da5f5ef0568dfacd542e, server=bd6592c37bb2,40087,1732486423713 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T22:14:50,965 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732486488674.cdb3383a4019da5f5ef0568dfacd542e., hostname=bd6592c37bb2,40087,1732486423713, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cdb3383a4019da5f5ef0568dfacd542e, server=bd6592c37bb2,40087,1732486423713 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T22:14:50,965 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40087 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cdb3383a4019da5f5ef0568dfacd542e, server=bd6592c37bb2,40087,1732486423713 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-24T22:14:50,965 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732486488674.cdb3383a4019da5f5ef0568dfacd542e., hostname=bd6592c37bb2,40087,1732486423713, seqNum=2 because the exception is null or not the one we care about 2024-11-24T22:14:50,965 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40087 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40806 deadline: 1732486550959, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cdb3383a4019da5f5ef0568dfacd542e, server=bd6592c37bb2,40087,1732486423713 2024-11-24T22:14:50,966 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732486488674.cdb3383a4019da5f5ef0568dfacd542e., hostname=bd6592c37bb2,40087,1732486423713, seqNum=2 , the old value is region=TestAcidGuarantees,,1732486488674.cdb3383a4019da5f5ef0568dfacd542e., hostname=bd6592c37bb2,40087,1732486423713, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cdb3383a4019da5f5ef0568dfacd542e, server=bd6592c37bb2,40087,1732486423713 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T22:14:50,966 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732486488674.cdb3383a4019da5f5ef0568dfacd542e., hostname=bd6592c37bb2,40087,1732486423713, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cdb3383a4019da5f5ef0568dfacd542e, server=bd6592c37bb2,40087,1732486423713 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T22:14:50,966 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732486488674.cdb3383a4019da5f5ef0568dfacd542e., hostname=bd6592c37bb2,40087,1732486423713, seqNum=2 because the exception is null or not the one we care about 2024-11-24T22:14:50,970 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/cdb3383a4019da5f5ef0568dfacd542e/.tmp/B/aff61553ec6940aa88ba531fe96ccd65 is 50, key is test_row_0/B:col10/1732486490888/Put/seqid=0 2024-11-24T22:14:50,981 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37701 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=60 2024-11-24T22:14:50,990 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45929 is added to blk_1073742070_1246 (size=12001) 2024-11-24T22:14:50,992 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=12 (bloomFilter=true), to=hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/cdb3383a4019da5f5ef0568dfacd542e/.tmp/B/aff61553ec6940aa88ba531fe96ccd65 2024-11-24T22:14:51,033 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/cdb3383a4019da5f5ef0568dfacd542e/.tmp/C/9af272d332df4b0792e8c07b7f46a69f is 50, key is test_row_0/C:col10/1732486490888/Put/seqid=0 2024-11-24T22:14:51,034 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40087 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cdb3383a4019da5f5ef0568dfacd542e, server=bd6592c37bb2,40087,1732486423713 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-24T22:14:51,034 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40087 {}] ipc.CallRunner(138): callId: 6 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40834 deadline: 1732486551031, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cdb3383a4019da5f5ef0568dfacd542e, server=bd6592c37bb2,40087,1732486423713 2024-11-24T22:14:51,035 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732486488674.cdb3383a4019da5f5ef0568dfacd542e., hostname=bd6592c37bb2,40087,1732486423713, seqNum=2 , the old value is region=TestAcidGuarantees,,1732486488674.cdb3383a4019da5f5ef0568dfacd542e., hostname=bd6592c37bb2,40087,1732486423713, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cdb3383a4019da5f5ef0568dfacd542e, server=bd6592c37bb2,40087,1732486423713 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T22:14:51,035 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732486488674.cdb3383a4019da5f5ef0568dfacd542e., hostname=bd6592c37bb2,40087,1732486423713, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cdb3383a4019da5f5ef0568dfacd542e, server=bd6592c37bb2,40087,1732486423713 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T22:14:51,035 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732486488674.cdb3383a4019da5f5ef0568dfacd542e., hostname=bd6592c37bb2,40087,1732486423713, seqNum=2 because the exception is null or not the one we care about 2024-11-24T22:14:51,035 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=40087 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=61 2024-11-24T22:14:51,035 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bd6592c37bb2:0-0 {event_type=RS_FLUSH_REGIONS, pid=61}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732486488674.cdb3383a4019da5f5ef0568dfacd542e. 2024-11-24T22:14:51,035 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bd6592c37bb2:0-0 {event_type=RS_FLUSH_REGIONS, pid=61}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732486488674.cdb3383a4019da5f5ef0568dfacd542e. as already flushing 2024-11-24T22:14:51,035 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bd6592c37bb2:0-0 {event_type=RS_FLUSH_REGIONS, pid=61}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732486488674.cdb3383a4019da5f5ef0568dfacd542e. 2024-11-24T22:14:51,036 ERROR [RS_FLUSH_OPERATIONS-regionserver/bd6592c37bb2:0-0 {event_type=RS_FLUSH_REGIONS, pid=61}] handler.RSProcedureHandler(58): pid=61 java.io.IOException: Unable to complete flush {ENCODED => cdb3383a4019da5f5ef0568dfacd542e, NAME => 'TestAcidGuarantees,,1732486488674.cdb3383a4019da5f5ef0568dfacd542e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T22:14:51,036 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bd6592c37bb2:0-0 {event_type=RS_FLUSH_REGIONS, pid=61}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=61 java.io.IOException: Unable to complete flush {ENCODED => cdb3383a4019da5f5ef0568dfacd542e, NAME => 'TestAcidGuarantees,,1732486488674.cdb3383a4019da5f5ef0568dfacd542e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T22:14:51,036 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37701 {}] master.HMaster(4177): Remote procedure failed, pid=61 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => cdb3383a4019da5f5ef0568dfacd542e, NAME => 'TestAcidGuarantees,,1732486488674.cdb3383a4019da5f5ef0568dfacd542e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => cdb3383a4019da5f5ef0568dfacd542e, NAME => 'TestAcidGuarantees,,1732486488674.cdb3383a4019da5f5ef0568dfacd542e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T22:14:51,049 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45929 is added to blk_1073742071_1247 (size=12001) 2024-11-24T22:14:51,050 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=12 (bloomFilter=true), to=hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/cdb3383a4019da5f5ef0568dfacd542e/.tmp/C/9af272d332df4b0792e8c07b7f46a69f 2024-11-24T22:14:51,068 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/cdb3383a4019da5f5ef0568dfacd542e/.tmp/A/4eb6e5cbfdd146468d5f9e4bee6f4417 as hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/cdb3383a4019da5f5ef0568dfacd542e/A/4eb6e5cbfdd146468d5f9e4bee6f4417 2024-11-24T22:14:51,073 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40087 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cdb3383a4019da5f5ef0568dfacd542e, server=bd6592c37bb2,40087,1732486423713 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-24T22:14:51,073 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40087 {}] ipc.CallRunner(138): callId: 12 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40772 deadline: 1732486551071, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cdb3383a4019da5f5ef0568dfacd542e, server=bd6592c37bb2,40087,1732486423713 2024-11-24T22:14:51,074 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732486488674.cdb3383a4019da5f5ef0568dfacd542e., hostname=bd6592c37bb2,40087,1732486423713, seqNum=2 , the old value is region=TestAcidGuarantees,,1732486488674.cdb3383a4019da5f5ef0568dfacd542e., hostname=bd6592c37bb2,40087,1732486423713, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cdb3383a4019da5f5ef0568dfacd542e, server=bd6592c37bb2,40087,1732486423713 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T22:14:51,074 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40087 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cdb3383a4019da5f5ef0568dfacd542e, server=bd6592c37bb2,40087,1732486423713 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-24T22:14:51,074 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732486488674.cdb3383a4019da5f5ef0568dfacd542e., hostname=bd6592c37bb2,40087,1732486423713, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cdb3383a4019da5f5ef0568dfacd542e, server=bd6592c37bb2,40087,1732486423713 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T22:14:51,074 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732486488674.cdb3383a4019da5f5ef0568dfacd542e., hostname=bd6592c37bb2,40087,1732486423713, seqNum=2 because the exception is null or not the one we care about 2024-11-24T22:14:51,074 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40087 {}] ipc.CallRunner(138): callId: 11 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40842 deadline: 1732486551071, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cdb3383a4019da5f5ef0568dfacd542e, server=bd6592c37bb2,40087,1732486423713 2024-11-24T22:14:51,074 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40087 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cdb3383a4019da5f5ef0568dfacd542e, server=bd6592c37bb2,40087,1732486423713 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-24T22:14:51,074 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732486488674.cdb3383a4019da5f5ef0568dfacd542e., hostname=bd6592c37bb2,40087,1732486423713, seqNum=2 , the old value is region=TestAcidGuarantees,,1732486488674.cdb3383a4019da5f5ef0568dfacd542e., hostname=bd6592c37bb2,40087,1732486423713, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cdb3383a4019da5f5ef0568dfacd542e, server=bd6592c37bb2,40087,1732486423713 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T22:14:51,074 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732486488674.cdb3383a4019da5f5ef0568dfacd542e., hostname=bd6592c37bb2,40087,1732486423713, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cdb3383a4019da5f5ef0568dfacd542e, server=bd6592c37bb2,40087,1732486423713 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T22:14:51,074 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40087 {}] ipc.CallRunner(138): callId: 11 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40816 deadline: 1732486551071, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cdb3383a4019da5f5ef0568dfacd542e, server=bd6592c37bb2,40087,1732486423713 2024-11-24T22:14:51,074 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732486488674.cdb3383a4019da5f5ef0568dfacd542e., hostname=bd6592c37bb2,40087,1732486423713, seqNum=2 because the exception is null or not the one we care about 2024-11-24T22:14:51,074 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40087 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cdb3383a4019da5f5ef0568dfacd542e, server=bd6592c37bb2,40087,1732486423713 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-24T22:14:51,075 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40087 {}] ipc.CallRunner(138): callId: 11 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40806 deadline: 1732486551071, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cdb3383a4019da5f5ef0568dfacd542e, server=bd6592c37bb2,40087,1732486423713 2024-11-24T22:14:51,075 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732486488674.cdb3383a4019da5f5ef0568dfacd542e., hostname=bd6592c37bb2,40087,1732486423713, seqNum=2 , the old value is region=TestAcidGuarantees,,1732486488674.cdb3383a4019da5f5ef0568dfacd542e., hostname=bd6592c37bb2,40087,1732486423713, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cdb3383a4019da5f5ef0568dfacd542e, server=bd6592c37bb2,40087,1732486423713 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T22:14:51,075 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732486488674.cdb3383a4019da5f5ef0568dfacd542e., hostname=bd6592c37bb2,40087,1732486423713, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cdb3383a4019da5f5ef0568dfacd542e, server=bd6592c37bb2,40087,1732486423713 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T22:14:51,075 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732486488674.cdb3383a4019da5f5ef0568dfacd542e., hostname=bd6592c37bb2,40087,1732486423713, seqNum=2 because the exception is null or not the one we care about 2024-11-24T22:14:51,075 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732486488674.cdb3383a4019da5f5ef0568dfacd542e., hostname=bd6592c37bb2,40087,1732486423713, seqNum=2 , the old value is region=TestAcidGuarantees,,1732486488674.cdb3383a4019da5f5ef0568dfacd542e., hostname=bd6592c37bb2,40087,1732486423713, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cdb3383a4019da5f5ef0568dfacd542e, server=bd6592c37bb2,40087,1732486423713 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T22:14:51,075 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732486488674.cdb3383a4019da5f5ef0568dfacd542e., hostname=bd6592c37bb2,40087,1732486423713, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cdb3383a4019da5f5ef0568dfacd542e, server=bd6592c37bb2,40087,1732486423713 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T22:14:51,075 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732486488674.cdb3383a4019da5f5ef0568dfacd542e., hostname=bd6592c37bb2,40087,1732486423713, seqNum=2 because the exception is null or not the one we care about 2024-11-24T22:14:51,077 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/cdb3383a4019da5f5ef0568dfacd542e/A/4eb6e5cbfdd146468d5f9e4bee6f4417, entries=150, sequenceid=12, filesize=11.7 K 2024-11-24T22:14:51,079 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/cdb3383a4019da5f5ef0568dfacd542e/.tmp/B/aff61553ec6940aa88ba531fe96ccd65 as hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/cdb3383a4019da5f5ef0568dfacd542e/B/aff61553ec6940aa88ba531fe96ccd65 2024-11-24T22:14:51,085 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/cdb3383a4019da5f5ef0568dfacd542e/B/aff61553ec6940aa88ba531fe96ccd65, entries=150, sequenceid=12, filesize=11.7 K 2024-11-24T22:14:51,086 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/cdb3383a4019da5f5ef0568dfacd542e/.tmp/C/9af272d332df4b0792e8c07b7f46a69f as hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/cdb3383a4019da5f5ef0568dfacd542e/C/9af272d332df4b0792e8c07b7f46a69f 2024-11-24T22:14:51,092 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/cdb3383a4019da5f5ef0568dfacd542e/C/9af272d332df4b0792e8c07b7f46a69f, entries=150, sequenceid=12, filesize=11.7 K 2024-11-24T22:14:51,093 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~53.67 KB/54960, heapSize ~160.08 KB/163920, currentSize=127.47 KB/130530 for cdb3383a4019da5f5ef0568dfacd542e in 204ms, sequenceid=12, compaction requested=false 2024-11-24T22:14:51,093 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for cdb3383a4019da5f5ef0568dfacd542e: 2024-11-24T22:14:51,189 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=40087 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=61 2024-11-24T22:14:51,189 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bd6592c37bb2:0-1 {event_type=RS_FLUSH_REGIONS, pid=61}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732486488674.cdb3383a4019da5f5ef0568dfacd542e. 2024-11-24T22:14:51,189 INFO [RS_FLUSH_OPERATIONS-regionserver/bd6592c37bb2:0-1 {event_type=RS_FLUSH_REGIONS, pid=61}] regionserver.HRegion(2902): Flushing cdb3383a4019da5f5ef0568dfacd542e 3/3 column families, dataSize=127.47 KB heapSize=365.06 KB 2024-11-24T22:14:51,190 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bd6592c37bb2:0-1 {event_type=RS_FLUSH_REGIONS, pid=61}] regionserver.CompactingMemStore(205): FLUSHING TO DISK cdb3383a4019da5f5ef0568dfacd542e, store=A 2024-11-24T22:14:51,190 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bd6592c37bb2:0-1 {event_type=RS_FLUSH_REGIONS, pid=61}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-24T22:14:51,190 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bd6592c37bb2:0-1 {event_type=RS_FLUSH_REGIONS, pid=61}] regionserver.CompactingMemStore(205): FLUSHING TO DISK cdb3383a4019da5f5ef0568dfacd542e, store=B 2024-11-24T22:14:51,190 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bd6592c37bb2:0-1 {event_type=RS_FLUSH_REGIONS, pid=61}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-24T22:14:51,190 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bd6592c37bb2:0-1 {event_type=RS_FLUSH_REGIONS, pid=61}] regionserver.CompactingMemStore(205): FLUSHING TO DISK cdb3383a4019da5f5ef0568dfacd542e, store=C 2024-11-24T22:14:51,190 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bd6592c37bb2:0-1 {event_type=RS_FLUSH_REGIONS, pid=61}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-24T22:14:51,191 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37701 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=60 2024-11-24T22:14:51,201 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bd6592c37bb2:0-1 {event_type=RS_FLUSH_REGIONS, pid=61}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/cdb3383a4019da5f5ef0568dfacd542e/.tmp/A/afbf7431336d498ab5f0bde061ac8ef3 is 50, key is test_row_0/A:col10/1732486490956/Put/seqid=0 2024-11-24T22:14:51,232 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45929 is added to blk_1073742072_1248 (size=12001) 2024-11-24T22:14:51,233 INFO [RS_FLUSH_OPERATIONS-regionserver/bd6592c37bb2:0-1 {event_type=RS_FLUSH_REGIONS, pid=61}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=42.49 KB at sequenceid=48 (bloomFilter=true), to=hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/cdb3383a4019da5f5ef0568dfacd542e/.tmp/A/afbf7431336d498ab5f0bde061ac8ef3 2024-11-24T22:14:51,244 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40087 {}] regionserver.HRegion(8855): Flush requested on cdb3383a4019da5f5ef0568dfacd542e 2024-11-24T22:14:51,244 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732486488674.cdb3383a4019da5f5ef0568dfacd542e. as already flushing 2024-11-24T22:14:51,245 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bd6592c37bb2:0-1 {event_type=RS_FLUSH_REGIONS, pid=61}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/cdb3383a4019da5f5ef0568dfacd542e/.tmp/B/760c5f08c89a4d1cb6849afadfb67357 is 50, key is test_row_0/B:col10/1732486490956/Put/seqid=0 2024-11-24T22:14:51,265 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45929 is added to blk_1073742073_1249 (size=12001) 2024-11-24T22:14:51,266 INFO [RS_FLUSH_OPERATIONS-regionserver/bd6592c37bb2:0-1 {event_type=RS_FLUSH_REGIONS, pid=61}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=42.49 KB at sequenceid=48 (bloomFilter=true), to=hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/cdb3383a4019da5f5ef0568dfacd542e/.tmp/B/760c5f08c89a4d1cb6849afadfb67357 2024-11-24T22:14:51,279 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bd6592c37bb2:0-1 {event_type=RS_FLUSH_REGIONS, pid=61}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/cdb3383a4019da5f5ef0568dfacd542e/.tmp/C/3a37abeb7cea46f495ee0f637ba2b9fb is 50, key is test_row_0/C:col10/1732486490956/Put/seqid=0 2024-11-24T22:14:51,284 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40087 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cdb3383a4019da5f5ef0568dfacd542e, server=bd6592c37bb2,40087,1732486423713 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-24T22:14:51,284 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40087 {}] ipc.CallRunner(138): callId: 13 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40772 deadline: 1732486551281, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cdb3383a4019da5f5ef0568dfacd542e, server=bd6592c37bb2,40087,1732486423713 2024-11-24T22:14:51,284 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40087 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cdb3383a4019da5f5ef0568dfacd542e, server=bd6592c37bb2,40087,1732486423713 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-24T22:14:51,284 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40087 {}] ipc.CallRunner(138): callId: 12 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40842 deadline: 1732486551281, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cdb3383a4019da5f5ef0568dfacd542e, server=bd6592c37bb2,40087,1732486423713 2024-11-24T22:14:51,284 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732486488674.cdb3383a4019da5f5ef0568dfacd542e., hostname=bd6592c37bb2,40087,1732486423713, seqNum=2 , the old value is region=TestAcidGuarantees,,1732486488674.cdb3383a4019da5f5ef0568dfacd542e., hostname=bd6592c37bb2,40087,1732486423713, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cdb3383a4019da5f5ef0568dfacd542e, server=bd6592c37bb2,40087,1732486423713 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T22:14:51,284 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732486488674.cdb3383a4019da5f5ef0568dfacd542e., hostname=bd6592c37bb2,40087,1732486423713, seqNum=2 , the old value is region=TestAcidGuarantees,,1732486488674.cdb3383a4019da5f5ef0568dfacd542e., hostname=bd6592c37bb2,40087,1732486423713, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cdb3383a4019da5f5ef0568dfacd542e, server=bd6592c37bb2,40087,1732486423713 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T22:14:51,284 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732486488674.cdb3383a4019da5f5ef0568dfacd542e., hostname=bd6592c37bb2,40087,1732486423713, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cdb3383a4019da5f5ef0568dfacd542e, server=bd6592c37bb2,40087,1732486423713 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T22:14:51,284 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732486488674.cdb3383a4019da5f5ef0568dfacd542e., hostname=bd6592c37bb2,40087,1732486423713, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cdb3383a4019da5f5ef0568dfacd542e, server=bd6592c37bb2,40087,1732486423713 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T22:14:51,285 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732486488674.cdb3383a4019da5f5ef0568dfacd542e., hostname=bd6592c37bb2,40087,1732486423713, seqNum=2 because the exception is null or not the one we care about 2024-11-24T22:14:51,285 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732486488674.cdb3383a4019da5f5ef0568dfacd542e., hostname=bd6592c37bb2,40087,1732486423713, seqNum=2 because the exception is null or not the one we care about 2024-11-24T22:14:51,285 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40087 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cdb3383a4019da5f5ef0568dfacd542e, server=bd6592c37bb2,40087,1732486423713 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-24T22:14:51,285 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40087 {}] ipc.CallRunner(138): callId: 12 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40806 deadline: 1732486551281, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cdb3383a4019da5f5ef0568dfacd542e, server=bd6592c37bb2,40087,1732486423713 2024-11-24T22:14:51,285 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40087 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cdb3383a4019da5f5ef0568dfacd542e, server=bd6592c37bb2,40087,1732486423713 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-24T22:14:51,285 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732486488674.cdb3383a4019da5f5ef0568dfacd542e., hostname=bd6592c37bb2,40087,1732486423713, seqNum=2 , the old value is region=TestAcidGuarantees,,1732486488674.cdb3383a4019da5f5ef0568dfacd542e., hostname=bd6592c37bb2,40087,1732486423713, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cdb3383a4019da5f5ef0568dfacd542e, server=bd6592c37bb2,40087,1732486423713 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T22:14:51,285 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732486488674.cdb3383a4019da5f5ef0568dfacd542e., hostname=bd6592c37bb2,40087,1732486423713, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cdb3383a4019da5f5ef0568dfacd542e, server=bd6592c37bb2,40087,1732486423713 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T22:14:51,285 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40087 {}] ipc.CallRunner(138): callId: 12 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40816 deadline: 1732486551281, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cdb3383a4019da5f5ef0568dfacd542e, server=bd6592c37bb2,40087,1732486423713 2024-11-24T22:14:51,285 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732486488674.cdb3383a4019da5f5ef0568dfacd542e., hostname=bd6592c37bb2,40087,1732486423713, seqNum=2 because the exception is null or not the one we care about 2024-11-24T22:14:51,286 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732486488674.cdb3383a4019da5f5ef0568dfacd542e., hostname=bd6592c37bb2,40087,1732486423713, seqNum=2 , the old value is region=TestAcidGuarantees,,1732486488674.cdb3383a4019da5f5ef0568dfacd542e., hostname=bd6592c37bb2,40087,1732486423713, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cdb3383a4019da5f5ef0568dfacd542e, server=bd6592c37bb2,40087,1732486423713 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T22:14:51,286 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732486488674.cdb3383a4019da5f5ef0568dfacd542e., hostname=bd6592c37bb2,40087,1732486423713, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cdb3383a4019da5f5ef0568dfacd542e, server=bd6592c37bb2,40087,1732486423713 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T22:14:51,286 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732486488674.cdb3383a4019da5f5ef0568dfacd542e., hostname=bd6592c37bb2,40087,1732486423713, seqNum=2 because the exception is null or not the one we care about 2024-11-24T22:14:51,287 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40087 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cdb3383a4019da5f5ef0568dfacd542e, server=bd6592c37bb2,40087,1732486423713 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-24T22:14:51,287 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40087 {}] ipc.CallRunner(138): callId: 15 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40834 deadline: 1732486551285, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cdb3383a4019da5f5ef0568dfacd542e, server=bd6592c37bb2,40087,1732486423713 2024-11-24T22:14:51,288 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732486488674.cdb3383a4019da5f5ef0568dfacd542e., hostname=bd6592c37bb2,40087,1732486423713, seqNum=2 , the old value is region=TestAcidGuarantees,,1732486488674.cdb3383a4019da5f5ef0568dfacd542e., hostname=bd6592c37bb2,40087,1732486423713, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cdb3383a4019da5f5ef0568dfacd542e, server=bd6592c37bb2,40087,1732486423713 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T22:14:51,288 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732486488674.cdb3383a4019da5f5ef0568dfacd542e., hostname=bd6592c37bb2,40087,1732486423713, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cdb3383a4019da5f5ef0568dfacd542e, server=bd6592c37bb2,40087,1732486423713 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T22:14:51,288 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732486488674.cdb3383a4019da5f5ef0568dfacd542e., hostname=bd6592c37bb2,40087,1732486423713, seqNum=2 because the exception is null or not the one we care about 2024-11-24T22:14:51,289 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45929 is added to blk_1073742074_1250 (size=12001) 2024-11-24T22:14:51,292 INFO [RS_FLUSH_OPERATIONS-regionserver/bd6592c37bb2:0-1 {event_type=RS_FLUSH_REGIONS, pid=61}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=42.49 KB at sequenceid=48 (bloomFilter=true), to=hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/cdb3383a4019da5f5ef0568dfacd542e/.tmp/C/3a37abeb7cea46f495ee0f637ba2b9fb 2024-11-24T22:14:51,296 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bd6592c37bb2:0-1 {event_type=RS_FLUSH_REGIONS, pid=61}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/cdb3383a4019da5f5ef0568dfacd542e/.tmp/A/afbf7431336d498ab5f0bde061ac8ef3 as hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/cdb3383a4019da5f5ef0568dfacd542e/A/afbf7431336d498ab5f0bde061ac8ef3 2024-11-24T22:14:51,299 INFO [RS_FLUSH_OPERATIONS-regionserver/bd6592c37bb2:0-1 {event_type=RS_FLUSH_REGIONS, pid=61}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/cdb3383a4019da5f5ef0568dfacd542e/A/afbf7431336d498ab5f0bde061ac8ef3, entries=150, sequenceid=48, filesize=11.7 K 2024-11-24T22:14:51,300 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bd6592c37bb2:0-1 {event_type=RS_FLUSH_REGIONS, pid=61}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/cdb3383a4019da5f5ef0568dfacd542e/.tmp/B/760c5f08c89a4d1cb6849afadfb67357 as hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/cdb3383a4019da5f5ef0568dfacd542e/B/760c5f08c89a4d1cb6849afadfb67357 2024-11-24T22:14:51,305 INFO [RS_FLUSH_OPERATIONS-regionserver/bd6592c37bb2:0-1 {event_type=RS_FLUSH_REGIONS, pid=61}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/cdb3383a4019da5f5ef0568dfacd542e/B/760c5f08c89a4d1cb6849afadfb67357, entries=150, sequenceid=48, filesize=11.7 K 2024-11-24T22:14:51,307 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bd6592c37bb2:0-1 {event_type=RS_FLUSH_REGIONS, pid=61}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/cdb3383a4019da5f5ef0568dfacd542e/.tmp/C/3a37abeb7cea46f495ee0f637ba2b9fb as hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/cdb3383a4019da5f5ef0568dfacd542e/C/3a37abeb7cea46f495ee0f637ba2b9fb 2024-11-24T22:14:51,311 INFO [RS_FLUSH_OPERATIONS-regionserver/bd6592c37bb2:0-1 {event_type=RS_FLUSH_REGIONS, pid=61}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/cdb3383a4019da5f5ef0568dfacd542e/C/3a37abeb7cea46f495ee0f637ba2b9fb, entries=150, sequenceid=48, filesize=11.7 K 2024-11-24T22:14:51,312 INFO [RS_FLUSH_OPERATIONS-regionserver/bd6592c37bb2:0-1 {event_type=RS_FLUSH_REGIONS, pid=61}] regionserver.HRegion(3140): Finished flush of dataSize ~127.47 KB/130530, heapSize ~365.02 KB/373776, currentSize=53.67 KB/54960 for cdb3383a4019da5f5ef0568dfacd542e in 123ms, sequenceid=48, compaction requested=false 2024-11-24T22:14:51,312 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bd6592c37bb2:0-1 {event_type=RS_FLUSH_REGIONS, pid=61}] regionserver.HRegion(2603): Flush status journal for cdb3383a4019da5f5ef0568dfacd542e: 2024-11-24T22:14:51,312 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bd6592c37bb2:0-1 {event_type=RS_FLUSH_REGIONS, pid=61}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732486488674.cdb3383a4019da5f5ef0568dfacd542e. 2024-11-24T22:14:51,312 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bd6592c37bb2:0-1 {event_type=RS_FLUSH_REGIONS, pid=61}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=61 2024-11-24T22:14:51,313 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37701 {}] master.HMaster(4169): Remote procedure done, pid=61 2024-11-24T22:14:51,315 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=61, resume processing ppid=60 2024-11-24T22:14:51,316 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=61, ppid=60, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 433 msec 2024-11-24T22:14:51,317 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=60, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=60, table=TestAcidGuarantees in 457 msec 2024-11-24T22:14:51,394 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40087 {}] regionserver.HRegion(8855): Flush requested on cdb3383a4019da5f5ef0568dfacd542e 2024-11-24T22:14:51,394 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing cdb3383a4019da5f5ef0568dfacd542e 3/3 column families, dataSize=60.38 KB heapSize=180.05 KB 2024-11-24T22:14:51,394 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK cdb3383a4019da5f5ef0568dfacd542e, store=A 2024-11-24T22:14:51,395 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-24T22:14:51,395 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK cdb3383a4019da5f5ef0568dfacd542e, store=B 2024-11-24T22:14:51,395 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-24T22:14:51,395 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK cdb3383a4019da5f5ef0568dfacd542e, store=C 2024-11-24T22:14:51,395 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-24T22:14:51,407 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/cdb3383a4019da5f5ef0568dfacd542e/.tmp/A/278445a736f74720b5568544cc19082a is 50, key is test_row_0/A:col10/1732486491392/Put/seqid=0 2024-11-24T22:14:51,418 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45929 is added to blk_1073742075_1251 (size=14341) 2024-11-24T22:14:51,492 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40087 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cdb3383a4019da5f5ef0568dfacd542e, server=bd6592c37bb2,40087,1732486423713 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-24T22:14:51,492 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40087 {}] ipc.CallRunner(138): callId: 34 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40834 deadline: 1732486551487, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cdb3383a4019da5f5ef0568dfacd542e, server=bd6592c37bb2,40087,1732486423713 2024-11-24T22:14:51,493 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732486488674.cdb3383a4019da5f5ef0568dfacd542e., hostname=bd6592c37bb2,40087,1732486423713, seqNum=2 , the old value is region=TestAcidGuarantees,,1732486488674.cdb3383a4019da5f5ef0568dfacd542e., hostname=bd6592c37bb2,40087,1732486423713, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cdb3383a4019da5f5ef0568dfacd542e, server=bd6592c37bb2,40087,1732486423713 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T22:14:51,493 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732486488674.cdb3383a4019da5f5ef0568dfacd542e., hostname=bd6592c37bb2,40087,1732486423713, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cdb3383a4019da5f5ef0568dfacd542e, server=bd6592c37bb2,40087,1732486423713 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T22:14:51,493 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732486488674.cdb3383a4019da5f5ef0568dfacd542e., hostname=bd6592c37bb2,40087,1732486423713, seqNum=2 because the exception is null or not the one we care about 2024-11-24T22:14:51,501 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37701 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=60 2024-11-24T22:14:51,501 INFO [RPCClient-NioEventLoopGroup-4-6 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-24T22:14:51,501 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37701 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-24T22:14:51,502 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37701 {}] procedure2.ProcedureExecutor(1139): Stored pid=62, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=62, table=TestAcidGuarantees 2024-11-24T22:14:51,503 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37701 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=62 2024-11-24T22:14:51,503 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=62, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=62, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-24T22:14:51,503 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=62, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=62, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-24T22:14:51,504 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=63, ppid=62, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-24T22:14:51,592 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40087 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cdb3383a4019da5f5ef0568dfacd542e, server=bd6592c37bb2,40087,1732486423713 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-24T22:14:51,592 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40087 {}] ipc.CallRunner(138): callId: 14 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40772 deadline: 1732486551591, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cdb3383a4019da5f5ef0568dfacd542e, server=bd6592c37bb2,40087,1732486423713 2024-11-24T22:14:51,593 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40087 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cdb3383a4019da5f5ef0568dfacd542e, server=bd6592c37bb2,40087,1732486423713 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-24T22:14:51,593 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732486488674.cdb3383a4019da5f5ef0568dfacd542e., hostname=bd6592c37bb2,40087,1732486423713, seqNum=2 , the old value is region=TestAcidGuarantees,,1732486488674.cdb3383a4019da5f5ef0568dfacd542e., hostname=bd6592c37bb2,40087,1732486423713, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cdb3383a4019da5f5ef0568dfacd542e, server=bd6592c37bb2,40087,1732486423713 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T22:14:51,593 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40087 {}] ipc.CallRunner(138): callId: 13 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40842 deadline: 1732486551591, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cdb3383a4019da5f5ef0568dfacd542e, server=bd6592c37bb2,40087,1732486423713 2024-11-24T22:14:51,593 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732486488674.cdb3383a4019da5f5ef0568dfacd542e., hostname=bd6592c37bb2,40087,1732486423713, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cdb3383a4019da5f5ef0568dfacd542e, server=bd6592c37bb2,40087,1732486423713 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T22:14:51,593 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732486488674.cdb3383a4019da5f5ef0568dfacd542e., hostname=bd6592c37bb2,40087,1732486423713, seqNum=2 because the exception is null or not the one we care about 2024-11-24T22:14:51,593 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40087 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cdb3383a4019da5f5ef0568dfacd542e, server=bd6592c37bb2,40087,1732486423713 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-24T22:14:51,593 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40087 {}] ipc.CallRunner(138): callId: 13 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40806 deadline: 1732486551591, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cdb3383a4019da5f5ef0568dfacd542e, server=bd6592c37bb2,40087,1732486423713 2024-11-24T22:14:51,593 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732486488674.cdb3383a4019da5f5ef0568dfacd542e., hostname=bd6592c37bb2,40087,1732486423713, seqNum=2 , the old value is region=TestAcidGuarantees,,1732486488674.cdb3383a4019da5f5ef0568dfacd542e., hostname=bd6592c37bb2,40087,1732486423713, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cdb3383a4019da5f5ef0568dfacd542e, server=bd6592c37bb2,40087,1732486423713 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T22:14:51,593 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732486488674.cdb3383a4019da5f5ef0568dfacd542e., hostname=bd6592c37bb2,40087,1732486423713, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cdb3383a4019da5f5ef0568dfacd542e, server=bd6592c37bb2,40087,1732486423713 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T22:14:51,593 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732486488674.cdb3383a4019da5f5ef0568dfacd542e., hostname=bd6592c37bb2,40087,1732486423713, seqNum=2 because the exception is null or not the one we care about 2024-11-24T22:14:51,593 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732486488674.cdb3383a4019da5f5ef0568dfacd542e., hostname=bd6592c37bb2,40087,1732486423713, seqNum=2 , the old value is region=TestAcidGuarantees,,1732486488674.cdb3383a4019da5f5ef0568dfacd542e., hostname=bd6592c37bb2,40087,1732486423713, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cdb3383a4019da5f5ef0568dfacd542e, server=bd6592c37bb2,40087,1732486423713 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T22:14:51,593 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732486488674.cdb3383a4019da5f5ef0568dfacd542e., hostname=bd6592c37bb2,40087,1732486423713, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cdb3383a4019da5f5ef0568dfacd542e, server=bd6592c37bb2,40087,1732486423713 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T22:14:51,593 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732486488674.cdb3383a4019da5f5ef0568dfacd542e., hostname=bd6592c37bb2,40087,1732486423713, seqNum=2 because the exception is null or not the one we care about 2024-11-24T22:14:51,595 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40087 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cdb3383a4019da5f5ef0568dfacd542e, server=bd6592c37bb2,40087,1732486423713 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-24T22:14:51,595 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40087 {}] ipc.CallRunner(138): callId: 13 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40816 deadline: 1732486551591, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cdb3383a4019da5f5ef0568dfacd542e, server=bd6592c37bb2,40087,1732486423713 2024-11-24T22:14:51,595 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732486488674.cdb3383a4019da5f5ef0568dfacd542e., hostname=bd6592c37bb2,40087,1732486423713, seqNum=2 , the old value is region=TestAcidGuarantees,,1732486488674.cdb3383a4019da5f5ef0568dfacd542e., hostname=bd6592c37bb2,40087,1732486423713, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cdb3383a4019da5f5ef0568dfacd542e, server=bd6592c37bb2,40087,1732486423713 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T22:14:51,595 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732486488674.cdb3383a4019da5f5ef0568dfacd542e., hostname=bd6592c37bb2,40087,1732486423713, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cdb3383a4019da5f5ef0568dfacd542e, server=bd6592c37bb2,40087,1732486423713 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T22:14:51,595 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732486488674.cdb3383a4019da5f5ef0568dfacd542e., hostname=bd6592c37bb2,40087,1732486423713, seqNum=2 because the exception is null or not the one we care about 2024-11-24T22:14:51,604 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40087 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cdb3383a4019da5f5ef0568dfacd542e, server=bd6592c37bb2,40087,1732486423713 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-24T22:14:51,604 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40087 {}] ipc.CallRunner(138): callId: 35 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40834 deadline: 1732486551601, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cdb3383a4019da5f5ef0568dfacd542e, server=bd6592c37bb2,40087,1732486423713 2024-11-24T22:14:51,604 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732486488674.cdb3383a4019da5f5ef0568dfacd542e., hostname=bd6592c37bb2,40087,1732486423713, seqNum=2 , the old value is region=TestAcidGuarantees,,1732486488674.cdb3383a4019da5f5ef0568dfacd542e., hostname=bd6592c37bb2,40087,1732486423713, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cdb3383a4019da5f5ef0568dfacd542e, server=bd6592c37bb2,40087,1732486423713 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T22:14:51,604 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732486488674.cdb3383a4019da5f5ef0568dfacd542e., hostname=bd6592c37bb2,40087,1732486423713, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cdb3383a4019da5f5ef0568dfacd542e, server=bd6592c37bb2,40087,1732486423713 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T22:14:51,604 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732486488674.cdb3383a4019da5f5ef0568dfacd542e., hostname=bd6592c37bb2,40087,1732486423713, seqNum=2 because the exception is null or not the one we care about 2024-11-24T22:14:51,610 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37701 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=62 2024-11-24T22:14:51,656 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=40087 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=63 2024-11-24T22:14:51,656 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bd6592c37bb2:0-2 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732486488674.cdb3383a4019da5f5ef0568dfacd542e. 2024-11-24T22:14:51,656 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bd6592c37bb2:0-2 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732486488674.cdb3383a4019da5f5ef0568dfacd542e. as already flushing 2024-11-24T22:14:51,656 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bd6592c37bb2:0-2 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732486488674.cdb3383a4019da5f5ef0568dfacd542e. 2024-11-24T22:14:51,656 ERROR [RS_FLUSH_OPERATIONS-regionserver/bd6592c37bb2:0-2 {event_type=RS_FLUSH_REGIONS, pid=63}] handler.RSProcedureHandler(58): pid=63 java.io.IOException: Unable to complete flush {ENCODED => cdb3383a4019da5f5ef0568dfacd542e, NAME => 'TestAcidGuarantees,,1732486488674.cdb3383a4019da5f5ef0568dfacd542e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T22:14:51,656 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bd6592c37bb2:0-2 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=63 java.io.IOException: Unable to complete flush {ENCODED => cdb3383a4019da5f5ef0568dfacd542e, NAME => 'TestAcidGuarantees,,1732486488674.cdb3383a4019da5f5ef0568dfacd542e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T22:14:51,657 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37701 {}] master.HMaster(4177): Remote procedure failed, pid=63 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => cdb3383a4019da5f5ef0568dfacd542e, NAME => 'TestAcidGuarantees,,1732486488674.cdb3383a4019da5f5ef0568dfacd542e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => cdb3383a4019da5f5ef0568dfacd542e, NAME => 'TestAcidGuarantees,,1732486488674.cdb3383a4019da5f5ef0568dfacd542e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T22:14:51,809 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=40087 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=63 2024-11-24T22:14:51,809 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bd6592c37bb2:0-0 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732486488674.cdb3383a4019da5f5ef0568dfacd542e. 2024-11-24T22:14:51,809 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bd6592c37bb2:0-0 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732486488674.cdb3383a4019da5f5ef0568dfacd542e. as already flushing 2024-11-24T22:14:51,809 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bd6592c37bb2:0-0 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732486488674.cdb3383a4019da5f5ef0568dfacd542e. 2024-11-24T22:14:51,809 ERROR [RS_FLUSH_OPERATIONS-regionserver/bd6592c37bb2:0-0 {event_type=RS_FLUSH_REGIONS, pid=63}] handler.RSProcedureHandler(58): pid=63 java.io.IOException: Unable to complete flush {ENCODED => cdb3383a4019da5f5ef0568dfacd542e, NAME => 'TestAcidGuarantees,,1732486488674.cdb3383a4019da5f5ef0568dfacd542e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T22:14:51,809 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bd6592c37bb2:0-0 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=63 java.io.IOException: Unable to complete flush {ENCODED => cdb3383a4019da5f5ef0568dfacd542e, NAME => 'TestAcidGuarantees,,1732486488674.cdb3383a4019da5f5ef0568dfacd542e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T22:14:51,810 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37701 {}] master.HMaster(4177): Remote procedure failed, pid=63 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => cdb3383a4019da5f5ef0568dfacd542e, NAME => 'TestAcidGuarantees,,1732486488674.cdb3383a4019da5f5ef0568dfacd542e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => cdb3383a4019da5f5ef0568dfacd542e, NAME => 'TestAcidGuarantees,,1732486488674.cdb3383a4019da5f5ef0568dfacd542e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T22:14:51,814 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40087 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cdb3383a4019da5f5ef0568dfacd542e, server=bd6592c37bb2,40087,1732486423713 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-24T22:14:51,814 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40087 {}] ipc.CallRunner(138): callId: 36 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40834 deadline: 1732486551811, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cdb3383a4019da5f5ef0568dfacd542e, server=bd6592c37bb2,40087,1732486423713 2024-11-24T22:14:51,814 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732486488674.cdb3383a4019da5f5ef0568dfacd542e., hostname=bd6592c37bb2,40087,1732486423713, seqNum=2 , the old value is region=TestAcidGuarantees,,1732486488674.cdb3383a4019da5f5ef0568dfacd542e., hostname=bd6592c37bb2,40087,1732486423713, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cdb3383a4019da5f5ef0568dfacd542e, server=bd6592c37bb2,40087,1732486423713 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T22:14:51,814 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732486488674.cdb3383a4019da5f5ef0568dfacd542e., hostname=bd6592c37bb2,40087,1732486423713, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cdb3383a4019da5f5ef0568dfacd542e, server=bd6592c37bb2,40087,1732486423713 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T22:14:51,814 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732486488674.cdb3383a4019da5f5ef0568dfacd542e., hostname=bd6592c37bb2,40087,1732486423713, seqNum=2 because the exception is null or not the one we care about 2024-11-24T22:14:51,821 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=60 (bloomFilter=true), to=hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/cdb3383a4019da5f5ef0568dfacd542e/.tmp/A/278445a736f74720b5568544cc19082a 2024-11-24T22:14:51,821 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37701 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=62 2024-11-24T22:14:51,829 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/cdb3383a4019da5f5ef0568dfacd542e/.tmp/B/18ce81563423421c8c6d5c60c19cb405 is 50, key is test_row_0/B:col10/1732486491392/Put/seqid=0 2024-11-24T22:14:51,838 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45929 is added to blk_1073742076_1252 (size=12001) 2024-11-24T22:14:51,838 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=60 (bloomFilter=true), to=hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/cdb3383a4019da5f5ef0568dfacd542e/.tmp/B/18ce81563423421c8c6d5c60c19cb405 2024-11-24T22:14:51,846 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/cdb3383a4019da5f5ef0568dfacd542e/.tmp/C/94f82c5bbcdd4208abaa8b61d53da1cb is 50, key is test_row_0/C:col10/1732486491392/Put/seqid=0 2024-11-24T22:14:51,850 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45929 is added to blk_1073742077_1253 (size=12001) 2024-11-24T22:14:51,962 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=40087 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=63 2024-11-24T22:14:51,963 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bd6592c37bb2:0-1 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732486488674.cdb3383a4019da5f5ef0568dfacd542e. 2024-11-24T22:14:51,963 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bd6592c37bb2:0-1 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732486488674.cdb3383a4019da5f5ef0568dfacd542e. as already flushing 2024-11-24T22:14:51,963 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bd6592c37bb2:0-1 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732486488674.cdb3383a4019da5f5ef0568dfacd542e. 2024-11-24T22:14:51,963 ERROR [RS_FLUSH_OPERATIONS-regionserver/bd6592c37bb2:0-1 {event_type=RS_FLUSH_REGIONS, pid=63}] handler.RSProcedureHandler(58): pid=63 java.io.IOException: Unable to complete flush {ENCODED => cdb3383a4019da5f5ef0568dfacd542e, NAME => 'TestAcidGuarantees,,1732486488674.cdb3383a4019da5f5ef0568dfacd542e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T22:14:51,963 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bd6592c37bb2:0-1 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=63 java.io.IOException: Unable to complete flush {ENCODED => cdb3383a4019da5f5ef0568dfacd542e, NAME => 'TestAcidGuarantees,,1732486488674.cdb3383a4019da5f5ef0568dfacd542e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T22:14:51,964 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37701 {}] master.HMaster(4177): Remote procedure failed, pid=63 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => cdb3383a4019da5f5ef0568dfacd542e, NAME => 'TestAcidGuarantees,,1732486488674.cdb3383a4019da5f5ef0568dfacd542e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => cdb3383a4019da5f5ef0568dfacd542e, NAME => 'TestAcidGuarantees,,1732486488674.cdb3383a4019da5f5ef0568dfacd542e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T22:14:52,104 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40087 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cdb3383a4019da5f5ef0568dfacd542e, server=bd6592c37bb2,40087,1732486423713 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-24T22:14:52,104 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40087 {}] ipc.CallRunner(138): callId: 15 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40772 deadline: 1732486552101, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cdb3383a4019da5f5ef0568dfacd542e, server=bd6592c37bb2,40087,1732486423713 2024-11-24T22:14:52,105 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40087 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cdb3383a4019da5f5ef0568dfacd542e, server=bd6592c37bb2,40087,1732486423713 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-24T22:14:52,105 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732486488674.cdb3383a4019da5f5ef0568dfacd542e., hostname=bd6592c37bb2,40087,1732486423713, seqNum=2 , the old value is region=TestAcidGuarantees,,1732486488674.cdb3383a4019da5f5ef0568dfacd542e., hostname=bd6592c37bb2,40087,1732486423713, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cdb3383a4019da5f5ef0568dfacd542e, server=bd6592c37bb2,40087,1732486423713 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T22:14:52,105 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40087 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cdb3383a4019da5f5ef0568dfacd542e, server=bd6592c37bb2,40087,1732486423713 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-24T22:14:52,105 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732486488674.cdb3383a4019da5f5ef0568dfacd542e., hostname=bd6592c37bb2,40087,1732486423713, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cdb3383a4019da5f5ef0568dfacd542e, server=bd6592c37bb2,40087,1732486423713 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T22:14:52,105 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40087 {}] ipc.CallRunner(138): callId: 14 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40816 deadline: 1732486552101, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cdb3383a4019da5f5ef0568dfacd542e, server=bd6592c37bb2,40087,1732486423713 2024-11-24T22:14:52,105 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732486488674.cdb3383a4019da5f5ef0568dfacd542e., hostname=bd6592c37bb2,40087,1732486423713, seqNum=2 because the exception is null or not the one we care about 2024-11-24T22:14:52,105 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40087 {}] ipc.CallRunner(138): callId: 14 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40842 deadline: 1732486552101, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cdb3383a4019da5f5ef0568dfacd542e, server=bd6592c37bb2,40087,1732486423713 2024-11-24T22:14:52,105 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732486488674.cdb3383a4019da5f5ef0568dfacd542e., hostname=bd6592c37bb2,40087,1732486423713, seqNum=2 , the old value is region=TestAcidGuarantees,,1732486488674.cdb3383a4019da5f5ef0568dfacd542e., hostname=bd6592c37bb2,40087,1732486423713, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cdb3383a4019da5f5ef0568dfacd542e, server=bd6592c37bb2,40087,1732486423713 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T22:14:52,105 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732486488674.cdb3383a4019da5f5ef0568dfacd542e., hostname=bd6592c37bb2,40087,1732486423713, seqNum=2 , the old value is region=TestAcidGuarantees,,1732486488674.cdb3383a4019da5f5ef0568dfacd542e., hostname=bd6592c37bb2,40087,1732486423713, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cdb3383a4019da5f5ef0568dfacd542e, server=bd6592c37bb2,40087,1732486423713 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T22:14:52,105 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40087 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cdb3383a4019da5f5ef0568dfacd542e, server=bd6592c37bb2,40087,1732486423713 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-24T22:14:52,105 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732486488674.cdb3383a4019da5f5ef0568dfacd542e., hostname=bd6592c37bb2,40087,1732486423713, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cdb3383a4019da5f5ef0568dfacd542e, server=bd6592c37bb2,40087,1732486423713 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T22:14:52,105 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732486488674.cdb3383a4019da5f5ef0568dfacd542e., hostname=bd6592c37bb2,40087,1732486423713, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cdb3383a4019da5f5ef0568dfacd542e, server=bd6592c37bb2,40087,1732486423713 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T22:14:52,105 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732486488674.cdb3383a4019da5f5ef0568dfacd542e., hostname=bd6592c37bb2,40087,1732486423713, seqNum=2 because the exception is null or not the one we care about 2024-11-24T22:14:52,105 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732486488674.cdb3383a4019da5f5ef0568dfacd542e., hostname=bd6592c37bb2,40087,1732486423713, seqNum=2 because the exception is null or not the one we care about 2024-11-24T22:14:52,105 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40087 {}] ipc.CallRunner(138): callId: 14 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40806 deadline: 1732486552101, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cdb3383a4019da5f5ef0568dfacd542e, server=bd6592c37bb2,40087,1732486423713 2024-11-24T22:14:52,105 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732486488674.cdb3383a4019da5f5ef0568dfacd542e., hostname=bd6592c37bb2,40087,1732486423713, seqNum=2 , the old value is region=TestAcidGuarantees,,1732486488674.cdb3383a4019da5f5ef0568dfacd542e., hostname=bd6592c37bb2,40087,1732486423713, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cdb3383a4019da5f5ef0568dfacd542e, server=bd6592c37bb2,40087,1732486423713 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T22:14:52,106 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732486488674.cdb3383a4019da5f5ef0568dfacd542e., hostname=bd6592c37bb2,40087,1732486423713, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cdb3383a4019da5f5ef0568dfacd542e, server=bd6592c37bb2,40087,1732486423713 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T22:14:52,106 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732486488674.cdb3383a4019da5f5ef0568dfacd542e., hostname=bd6592c37bb2,40087,1732486423713, seqNum=2 because the exception is null or not the one we care about 2024-11-24T22:14:52,116 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=40087 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=63 2024-11-24T22:14:52,116 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bd6592c37bb2:0-2 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732486488674.cdb3383a4019da5f5ef0568dfacd542e. 2024-11-24T22:14:52,117 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bd6592c37bb2:0-2 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732486488674.cdb3383a4019da5f5ef0568dfacd542e. as already flushing 2024-11-24T22:14:52,117 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bd6592c37bb2:0-2 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732486488674.cdb3383a4019da5f5ef0568dfacd542e. 2024-11-24T22:14:52,117 ERROR [RS_FLUSH_OPERATIONS-regionserver/bd6592c37bb2:0-2 {event_type=RS_FLUSH_REGIONS, pid=63}] handler.RSProcedureHandler(58): pid=63 java.io.IOException: Unable to complete flush {ENCODED => cdb3383a4019da5f5ef0568dfacd542e, NAME => 'TestAcidGuarantees,,1732486488674.cdb3383a4019da5f5ef0568dfacd542e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T22:14:52,117 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bd6592c37bb2:0-2 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=63 java.io.IOException: Unable to complete flush {ENCODED => cdb3383a4019da5f5ef0568dfacd542e, NAME => 'TestAcidGuarantees,,1732486488674.cdb3383a4019da5f5ef0568dfacd542e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T22:14:52,117 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37701 {}] master.HMaster(4177): Remote procedure failed, pid=63 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => cdb3383a4019da5f5ef0568dfacd542e, NAME => 'TestAcidGuarantees,,1732486488674.cdb3383a4019da5f5ef0568dfacd542e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => cdb3383a4019da5f5ef0568dfacd542e, NAME => 'TestAcidGuarantees,,1732486488674.cdb3383a4019da5f5ef0568dfacd542e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T22:14:52,123 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40087 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cdb3383a4019da5f5ef0568dfacd542e, server=bd6592c37bb2,40087,1732486423713 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-24T22:14:52,124 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40087 {}] ipc.CallRunner(138): callId: 37 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40834 deadline: 1732486552120, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cdb3383a4019da5f5ef0568dfacd542e, server=bd6592c37bb2,40087,1732486423713 2024-11-24T22:14:52,124 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732486488674.cdb3383a4019da5f5ef0568dfacd542e., hostname=bd6592c37bb2,40087,1732486423713, seqNum=2 , the old value is region=TestAcidGuarantees,,1732486488674.cdb3383a4019da5f5ef0568dfacd542e., hostname=bd6592c37bb2,40087,1732486423713, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cdb3383a4019da5f5ef0568dfacd542e, server=bd6592c37bb2,40087,1732486423713 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T22:14:52,124 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732486488674.cdb3383a4019da5f5ef0568dfacd542e., hostname=bd6592c37bb2,40087,1732486423713, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cdb3383a4019da5f5ef0568dfacd542e, server=bd6592c37bb2,40087,1732486423713 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T22:14:52,124 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732486488674.cdb3383a4019da5f5ef0568dfacd542e., hostname=bd6592c37bb2,40087,1732486423713, seqNum=2 because the exception is null or not the one we care about 2024-11-24T22:14:52,131 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37701 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=62 2024-11-24T22:14:52,251 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=60 (bloomFilter=true), to=hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/cdb3383a4019da5f5ef0568dfacd542e/.tmp/C/94f82c5bbcdd4208abaa8b61d53da1cb 2024-11-24T22:14:52,256 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/cdb3383a4019da5f5ef0568dfacd542e/.tmp/A/278445a736f74720b5568544cc19082a as hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/cdb3383a4019da5f5ef0568dfacd542e/A/278445a736f74720b5568544cc19082a 2024-11-24T22:14:52,259 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/cdb3383a4019da5f5ef0568dfacd542e/A/278445a736f74720b5568544cc19082a, entries=200, sequenceid=60, filesize=14.0 K 2024-11-24T22:14:52,260 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/cdb3383a4019da5f5ef0568dfacd542e/.tmp/B/18ce81563423421c8c6d5c60c19cb405 as hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/cdb3383a4019da5f5ef0568dfacd542e/B/18ce81563423421c8c6d5c60c19cb405 2024-11-24T22:14:52,264 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/cdb3383a4019da5f5ef0568dfacd542e/B/18ce81563423421c8c6d5c60c19cb405, entries=150, sequenceid=60, filesize=11.7 K 2024-11-24T22:14:52,264 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/cdb3383a4019da5f5ef0568dfacd542e/.tmp/C/94f82c5bbcdd4208abaa8b61d53da1cb as hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/cdb3383a4019da5f5ef0568dfacd542e/C/94f82c5bbcdd4208abaa8b61d53da1cb 2024-11-24T22:14:52,268 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/cdb3383a4019da5f5ef0568dfacd542e/C/94f82c5bbcdd4208abaa8b61d53da1cb, entries=150, sequenceid=60, filesize=11.7 K 2024-11-24T22:14:52,269 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~60.38 KB/61830, heapSize ~180 KB/184320, currentSize=114.05 KB/116790 for cdb3383a4019da5f5ef0568dfacd542e in 874ms, sequenceid=60, compaction requested=true 2024-11-24T22:14:52,269 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for cdb3383a4019da5f5ef0568dfacd542e: 2024-11-24T22:14:52,269 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store cdb3383a4019da5f5ef0568dfacd542e:A, priority=-2147483648, current under compaction store size is 1 2024-11-24T22:14:52,269 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-24T22:14:52,269 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store cdb3383a4019da5f5ef0568dfacd542e:B, priority=-2147483648, current under compaction store size is 2 2024-11-24T22:14:52,269 DEBUG [RS:0;bd6592c37bb2:40087-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-24T22:14:52,269 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-24T22:14:52,269 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store cdb3383a4019da5f5ef0568dfacd542e:C, priority=-2147483648, current under compaction store size is 3 2024-11-24T22:14:52,269 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-24T22:14:52,269 DEBUG [RS:0;bd6592c37bb2:40087-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-24T22:14:52,269 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=40087 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=63 2024-11-24T22:14:52,270 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bd6592c37bb2:0-0 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732486488674.cdb3383a4019da5f5ef0568dfacd542e. 2024-11-24T22:14:52,270 INFO [RS_FLUSH_OPERATIONS-regionserver/bd6592c37bb2:0-0 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.HRegion(2902): Flushing cdb3383a4019da5f5ef0568dfacd542e 3/3 column families, dataSize=114.05 KB heapSize=339.42 KB 2024-11-24T22:14:52,270 DEBUG [RS:0;bd6592c37bb2:40087-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 38343 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-24T22:14:52,270 DEBUG [RS:0;bd6592c37bb2:40087-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36003 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-24T22:14:52,270 DEBUG [RS:0;bd6592c37bb2:40087-shortCompactions-0 {}] regionserver.HStore(1541): cdb3383a4019da5f5ef0568dfacd542e/A is initiating minor compaction (all files) 2024-11-24T22:14:52,270 DEBUG [RS:0;bd6592c37bb2:40087-longCompactions-0 {}] regionserver.HStore(1541): cdb3383a4019da5f5ef0568dfacd542e/B is initiating minor compaction (all files) 2024-11-24T22:14:52,270 INFO [RS:0;bd6592c37bb2:40087-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of cdb3383a4019da5f5ef0568dfacd542e/B in TestAcidGuarantees,,1732486488674.cdb3383a4019da5f5ef0568dfacd542e. 2024-11-24T22:14:52,270 INFO [RS:0;bd6592c37bb2:40087-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of cdb3383a4019da5f5ef0568dfacd542e/A in TestAcidGuarantees,,1732486488674.cdb3383a4019da5f5ef0568dfacd542e. 2024-11-24T22:14:52,270 INFO [RS:0;bd6592c37bb2:40087-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/cdb3383a4019da5f5ef0568dfacd542e/B/aff61553ec6940aa88ba531fe96ccd65, hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/cdb3383a4019da5f5ef0568dfacd542e/B/760c5f08c89a4d1cb6849afadfb67357, hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/cdb3383a4019da5f5ef0568dfacd542e/B/18ce81563423421c8c6d5c60c19cb405] into tmpdir=hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/cdb3383a4019da5f5ef0568dfacd542e/.tmp, totalSize=35.2 K 2024-11-24T22:14:52,270 INFO [RS:0;bd6592c37bb2:40087-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/cdb3383a4019da5f5ef0568dfacd542e/A/4eb6e5cbfdd146468d5f9e4bee6f4417, hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/cdb3383a4019da5f5ef0568dfacd542e/A/afbf7431336d498ab5f0bde061ac8ef3, hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/cdb3383a4019da5f5ef0568dfacd542e/A/278445a736f74720b5568544cc19082a] into tmpdir=hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/cdb3383a4019da5f5ef0568dfacd542e/.tmp, totalSize=37.4 K 2024-11-24T22:14:52,270 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bd6592c37bb2:0-0 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.CompactingMemStore(205): FLUSHING TO DISK cdb3383a4019da5f5ef0568dfacd542e, store=A 2024-11-24T22:14:52,270 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bd6592c37bb2:0-0 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-24T22:14:52,270 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bd6592c37bb2:0-0 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.CompactingMemStore(205): FLUSHING TO DISK cdb3383a4019da5f5ef0568dfacd542e, store=B 2024-11-24T22:14:52,270 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bd6592c37bb2:0-0 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-24T22:14:52,270 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bd6592c37bb2:0-0 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.CompactingMemStore(205): FLUSHING TO DISK cdb3383a4019da5f5ef0568dfacd542e, store=C 2024-11-24T22:14:52,270 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bd6592c37bb2:0-0 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-24T22:14:52,270 DEBUG [RS:0;bd6592c37bb2:40087-shortCompactions-0 {}] compactions.Compactor(225): Compacting 4eb6e5cbfdd146468d5f9e4bee6f4417, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=12, earliestPutTs=1732486490885 2024-11-24T22:14:52,270 DEBUG [RS:0;bd6592c37bb2:40087-longCompactions-0 {}] compactions.Compactor(225): Compacting aff61553ec6940aa88ba531fe96ccd65, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=12, earliestPutTs=1732486490885 2024-11-24T22:14:52,271 DEBUG [RS:0;bd6592c37bb2:40087-shortCompactions-0 {}] compactions.Compactor(225): Compacting afbf7431336d498ab5f0bde061ac8ef3, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=48, earliestPutTs=1732486490953 2024-11-24T22:14:52,271 DEBUG [RS:0;bd6592c37bb2:40087-longCompactions-0 {}] compactions.Compactor(225): Compacting 760c5f08c89a4d1cb6849afadfb67357, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=48, earliestPutTs=1732486490953 2024-11-24T22:14:52,271 DEBUG [RS:0;bd6592c37bb2:40087-shortCompactions-0 {}] compactions.Compactor(225): Compacting 278445a736f74720b5568544cc19082a, keycount=200, bloomtype=ROW, size=14.0 K, encoding=NONE, compression=NONE, seqNum=60, earliestPutTs=1732486491249 2024-11-24T22:14:52,271 DEBUG [RS:0;bd6592c37bb2:40087-longCompactions-0 {}] compactions.Compactor(225): Compacting 18ce81563423421c8c6d5c60c19cb405, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=60, earliestPutTs=1732486491249 2024-11-24T22:14:52,274 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bd6592c37bb2:0-0 {event_type=RS_FLUSH_REGIONS, pid=63}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/cdb3383a4019da5f5ef0568dfacd542e/.tmp/A/aa7e9dc47e4f4e2ebe297d6e390500bd is 50, key is test_row_0/A:col10/1732486491473/Put/seqid=0 2024-11-24T22:14:52,278 INFO [RS:0;bd6592c37bb2:40087-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): cdb3383a4019da5f5ef0568dfacd542e#B#compaction#205 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-24T22:14:52,278 DEBUG [RS:0;bd6592c37bb2:40087-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/cdb3383a4019da5f5ef0568dfacd542e/.tmp/B/e5b76e9a2c6b4facac53b583784c0fb7 is 50, key is test_row_0/B:col10/1732486491392/Put/seqid=0 2024-11-24T22:14:52,280 INFO [RS:0;bd6592c37bb2:40087-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): cdb3383a4019da5f5ef0568dfacd542e#A#compaction#206 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-24T22:14:52,281 DEBUG [RS:0;bd6592c37bb2:40087-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/cdb3383a4019da5f5ef0568dfacd542e/.tmp/A/7f34a3a979ba46fb954a392c6945ce2f is 50, key is test_row_0/A:col10/1732486491392/Put/seqid=0 2024-11-24T22:14:52,282 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45929 is added to blk_1073742078_1254 (size=12001) 2024-11-24T22:14:52,282 INFO [RS_FLUSH_OPERATIONS-regionserver/bd6592c37bb2:0-0 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=38.02 KB at sequenceid=80 (bloomFilter=true), to=hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/cdb3383a4019da5f5ef0568dfacd542e/.tmp/A/aa7e9dc47e4f4e2ebe297d6e390500bd 2024-11-24T22:14:52,290 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45929 is added to blk_1073742080_1256 (size=12104) 2024-11-24T22:14:52,290 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45929 is added to blk_1073742079_1255 (size=12104) 2024-11-24T22:14:52,292 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bd6592c37bb2:0-0 {event_type=RS_FLUSH_REGIONS, pid=63}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/cdb3383a4019da5f5ef0568dfacd542e/.tmp/B/3068e6ac5958446782debae8e550dc98 is 50, key is test_row_0/B:col10/1732486491473/Put/seqid=0 2024-11-24T22:14:52,296 DEBUG [RS:0;bd6592c37bb2:40087-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/cdb3383a4019da5f5ef0568dfacd542e/.tmp/B/e5b76e9a2c6b4facac53b583784c0fb7 as hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/cdb3383a4019da5f5ef0568dfacd542e/B/e5b76e9a2c6b4facac53b583784c0fb7 2024-11-24T22:14:52,301 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45929 is added to blk_1073742081_1257 (size=12001) 2024-11-24T22:14:52,302 INFO [RS:0;bd6592c37bb2:40087-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in cdb3383a4019da5f5ef0568dfacd542e/B of cdb3383a4019da5f5ef0568dfacd542e into e5b76e9a2c6b4facac53b583784c0fb7(size=11.8 K), total size for store is 11.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-24T22:14:52,302 DEBUG [RS:0;bd6592c37bb2:40087-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for cdb3383a4019da5f5ef0568dfacd542e: 2024-11-24T22:14:52,302 INFO [RS:0;bd6592c37bb2:40087-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732486488674.cdb3383a4019da5f5ef0568dfacd542e., storeName=cdb3383a4019da5f5ef0568dfacd542e/B, priority=13, startTime=1732486492269; duration=0sec 2024-11-24T22:14:52,302 DEBUG [RS:0;bd6592c37bb2:40087-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-24T22:14:52,302 DEBUG [RS:0;bd6592c37bb2:40087-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: cdb3383a4019da5f5ef0568dfacd542e:B 2024-11-24T22:14:52,303 DEBUG [RS:0;bd6592c37bb2:40087-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-24T22:14:52,303 INFO [RS_FLUSH_OPERATIONS-regionserver/bd6592c37bb2:0-0 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=38.02 KB at sequenceid=80 (bloomFilter=true), to=hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/cdb3383a4019da5f5ef0568dfacd542e/.tmp/B/3068e6ac5958446782debae8e550dc98 2024-11-24T22:14:52,304 DEBUG [RS:0;bd6592c37bb2:40087-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36003 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-24T22:14:52,304 DEBUG [RS:0;bd6592c37bb2:40087-longCompactions-0 {}] regionserver.HStore(1541): cdb3383a4019da5f5ef0568dfacd542e/C is initiating minor compaction (all files) 2024-11-24T22:14:52,305 INFO [RS:0;bd6592c37bb2:40087-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of cdb3383a4019da5f5ef0568dfacd542e/C in TestAcidGuarantees,,1732486488674.cdb3383a4019da5f5ef0568dfacd542e. 2024-11-24T22:14:52,305 INFO [RS:0;bd6592c37bb2:40087-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/cdb3383a4019da5f5ef0568dfacd542e/C/9af272d332df4b0792e8c07b7f46a69f, hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/cdb3383a4019da5f5ef0568dfacd542e/C/3a37abeb7cea46f495ee0f637ba2b9fb, hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/cdb3383a4019da5f5ef0568dfacd542e/C/94f82c5bbcdd4208abaa8b61d53da1cb] into tmpdir=hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/cdb3383a4019da5f5ef0568dfacd542e/.tmp, totalSize=35.2 K 2024-11-24T22:14:52,307 DEBUG [RS:0;bd6592c37bb2:40087-longCompactions-0 {}] compactions.Compactor(225): Compacting 9af272d332df4b0792e8c07b7f46a69f, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=12, earliestPutTs=1732486490885 2024-11-24T22:14:52,308 DEBUG [RS:0;bd6592c37bb2:40087-longCompactions-0 {}] compactions.Compactor(225): Compacting 3a37abeb7cea46f495ee0f637ba2b9fb, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=48, earliestPutTs=1732486490953 2024-11-24T22:14:52,310 DEBUG [RS:0;bd6592c37bb2:40087-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/cdb3383a4019da5f5ef0568dfacd542e/.tmp/A/7f34a3a979ba46fb954a392c6945ce2f as hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/cdb3383a4019da5f5ef0568dfacd542e/A/7f34a3a979ba46fb954a392c6945ce2f 2024-11-24T22:14:52,310 DEBUG [RS:0;bd6592c37bb2:40087-longCompactions-0 {}] compactions.Compactor(225): Compacting 94f82c5bbcdd4208abaa8b61d53da1cb, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=60, earliestPutTs=1732486491249 2024-11-24T22:14:52,315 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bd6592c37bb2:0-0 {event_type=RS_FLUSH_REGIONS, pid=63}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/cdb3383a4019da5f5ef0568dfacd542e/.tmp/C/54172941587641b39b3076d159e66605 is 50, key is test_row_0/C:col10/1732486491473/Put/seqid=0 2024-11-24T22:14:52,319 INFO [RS:0;bd6592c37bb2:40087-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in cdb3383a4019da5f5ef0568dfacd542e/A of cdb3383a4019da5f5ef0568dfacd542e into 7f34a3a979ba46fb954a392c6945ce2f(size=11.8 K), total size for store is 11.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-24T22:14:52,319 DEBUG [RS:0;bd6592c37bb2:40087-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for cdb3383a4019da5f5ef0568dfacd542e: 2024-11-24T22:14:52,319 INFO [RS:0;bd6592c37bb2:40087-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732486488674.cdb3383a4019da5f5ef0568dfacd542e., storeName=cdb3383a4019da5f5ef0568dfacd542e/A, priority=13, startTime=1732486492269; duration=0sec 2024-11-24T22:14:52,319 DEBUG [RS:0;bd6592c37bb2:40087-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-24T22:14:52,319 DEBUG [RS:0;bd6592c37bb2:40087-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: cdb3383a4019da5f5ef0568dfacd542e:A 2024-11-24T22:14:52,325 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45929 is added to blk_1073742082_1258 (size=12001) 2024-11-24T22:14:52,326 INFO [RS:0;bd6592c37bb2:40087-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): cdb3383a4019da5f5ef0568dfacd542e#C#compaction#209 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-24T22:14:52,327 DEBUG [RS:0;bd6592c37bb2:40087-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/cdb3383a4019da5f5ef0568dfacd542e/.tmp/C/73ebb44821fc486aa70667b86beeabc6 is 50, key is test_row_0/C:col10/1732486491392/Put/seqid=0 2024-11-24T22:14:52,344 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45929 is added to blk_1073742083_1259 (size=12104) 2024-11-24T22:14:52,633 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40087 {}] regionserver.HRegion(8855): Flush requested on cdb3383a4019da5f5ef0568dfacd542e 2024-11-24T22:14:52,633 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732486488674.cdb3383a4019da5f5ef0568dfacd542e. as already flushing 2024-11-24T22:14:52,640 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37701 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=62 2024-11-24T22:14:52,692 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40087 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cdb3383a4019da5f5ef0568dfacd542e, server=bd6592c37bb2,40087,1732486423713 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-24T22:14:52,692 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40087 {}] ipc.CallRunner(138): callId: 47 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40834 deadline: 1732486552688, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cdb3383a4019da5f5ef0568dfacd542e, server=bd6592c37bb2,40087,1732486423713 2024-11-24T22:14:52,693 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732486488674.cdb3383a4019da5f5ef0568dfacd542e., hostname=bd6592c37bb2,40087,1732486423713, seqNum=2 , the old value is region=TestAcidGuarantees,,1732486488674.cdb3383a4019da5f5ef0568dfacd542e., hostname=bd6592c37bb2,40087,1732486423713, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cdb3383a4019da5f5ef0568dfacd542e, server=bd6592c37bb2,40087,1732486423713 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T22:14:52,693 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732486488674.cdb3383a4019da5f5ef0568dfacd542e., hostname=bd6592c37bb2,40087,1732486423713, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cdb3383a4019da5f5ef0568dfacd542e, server=bd6592c37bb2,40087,1732486423713 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T22:14:52,693 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732486488674.cdb3383a4019da5f5ef0568dfacd542e., hostname=bd6592c37bb2,40087,1732486423713, seqNum=2 because the exception is null or not the one we care about 2024-11-24T22:14:52,726 INFO [RS_FLUSH_OPERATIONS-regionserver/bd6592c37bb2:0-0 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=38.02 KB at sequenceid=80 (bloomFilter=true), to=hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/cdb3383a4019da5f5ef0568dfacd542e/.tmp/C/54172941587641b39b3076d159e66605 2024-11-24T22:14:52,732 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bd6592c37bb2:0-0 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/cdb3383a4019da5f5ef0568dfacd542e/.tmp/A/aa7e9dc47e4f4e2ebe297d6e390500bd as hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/cdb3383a4019da5f5ef0568dfacd542e/A/aa7e9dc47e4f4e2ebe297d6e390500bd 2024-11-24T22:14:52,736 INFO [RS_FLUSH_OPERATIONS-regionserver/bd6592c37bb2:0-0 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/cdb3383a4019da5f5ef0568dfacd542e/A/aa7e9dc47e4f4e2ebe297d6e390500bd, entries=150, sequenceid=80, filesize=11.7 K 2024-11-24T22:14:52,737 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bd6592c37bb2:0-0 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/cdb3383a4019da5f5ef0568dfacd542e/.tmp/B/3068e6ac5958446782debae8e550dc98 as hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/cdb3383a4019da5f5ef0568dfacd542e/B/3068e6ac5958446782debae8e550dc98 2024-11-24T22:14:52,741 INFO [RS_FLUSH_OPERATIONS-regionserver/bd6592c37bb2:0-0 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/cdb3383a4019da5f5ef0568dfacd542e/B/3068e6ac5958446782debae8e550dc98, entries=150, sequenceid=80, filesize=11.7 K 2024-11-24T22:14:52,742 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bd6592c37bb2:0-0 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/cdb3383a4019da5f5ef0568dfacd542e/.tmp/C/54172941587641b39b3076d159e66605 as hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/cdb3383a4019da5f5ef0568dfacd542e/C/54172941587641b39b3076d159e66605 2024-11-24T22:14:52,746 DEBUG [RS:0;bd6592c37bb2:40087-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/cdb3383a4019da5f5ef0568dfacd542e/.tmp/C/73ebb44821fc486aa70667b86beeabc6 as hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/cdb3383a4019da5f5ef0568dfacd542e/C/73ebb44821fc486aa70667b86beeabc6 2024-11-24T22:14:52,751 INFO [RS_FLUSH_OPERATIONS-regionserver/bd6592c37bb2:0-0 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/cdb3383a4019da5f5ef0568dfacd542e/C/54172941587641b39b3076d159e66605, entries=150, sequenceid=80, filesize=11.7 K 2024-11-24T22:14:52,752 INFO [RS:0;bd6592c37bb2:40087-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in cdb3383a4019da5f5ef0568dfacd542e/C of cdb3383a4019da5f5ef0568dfacd542e into 73ebb44821fc486aa70667b86beeabc6(size=11.8 K), total size for store is 23.5 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-24T22:14:52,752 DEBUG [RS:0;bd6592c37bb2:40087-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for cdb3383a4019da5f5ef0568dfacd542e: 2024-11-24T22:14:52,752 INFO [RS:0;bd6592c37bb2:40087-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732486488674.cdb3383a4019da5f5ef0568dfacd542e., storeName=cdb3383a4019da5f5ef0568dfacd542e/C, priority=13, startTime=1732486492269; duration=0sec 2024-11-24T22:14:52,752 DEBUG [RS:0;bd6592c37bb2:40087-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-24T22:14:52,752 DEBUG [RS:0;bd6592c37bb2:40087-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: cdb3383a4019da5f5ef0568dfacd542e:C 2024-11-24T22:14:52,753 INFO [RS_FLUSH_OPERATIONS-regionserver/bd6592c37bb2:0-0 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.HRegion(3140): Finished flush of dataSize ~114.05 KB/116790, heapSize ~339.38 KB/347520, currentSize=60.38 KB/61830 for cdb3383a4019da5f5ef0568dfacd542e in 483ms, sequenceid=80, compaction requested=false 2024-11-24T22:14:52,753 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bd6592c37bb2:0-0 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.HRegion(2603): Flush status journal for cdb3383a4019da5f5ef0568dfacd542e: 2024-11-24T22:14:52,753 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bd6592c37bb2:0-0 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732486488674.cdb3383a4019da5f5ef0568dfacd542e. 2024-11-24T22:14:52,753 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bd6592c37bb2:0-0 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=63 2024-11-24T22:14:52,753 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37701 {}] master.HMaster(4169): Remote procedure done, pid=63 2024-11-24T22:14:52,757 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=63, resume processing ppid=62 2024-11-24T22:14:52,757 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=63, ppid=62, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.2500 sec 2024-11-24T22:14:52,765 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=62, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=62, table=TestAcidGuarantees in 1.2600 sec 2024-11-24T22:14:52,815 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40087 {}] regionserver.HRegion(8855): Flush requested on cdb3383a4019da5f5ef0568dfacd542e 2024-11-24T22:14:52,815 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing cdb3383a4019da5f5ef0568dfacd542e 3/3 column families, dataSize=67.09 KB heapSize=199.97 KB 2024-11-24T22:14:52,815 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK cdb3383a4019da5f5ef0568dfacd542e, store=A 2024-11-24T22:14:52,815 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-24T22:14:52,815 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK cdb3383a4019da5f5ef0568dfacd542e, store=B 2024-11-24T22:14:52,815 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-24T22:14:52,815 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK cdb3383a4019da5f5ef0568dfacd542e, store=C 2024-11-24T22:14:52,815 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-24T22:14:52,820 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/cdb3383a4019da5f5ef0568dfacd542e/.tmp/A/516acb04f2084f5ab2fb65855735bfac is 50, key is test_row_0/A:col10/1732486492687/Put/seqid=0 2024-11-24T22:14:52,832 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45929 is added to blk_1073742084_1260 (size=14341) 2024-11-24T22:14:52,837 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=96 (bloomFilter=true), to=hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/cdb3383a4019da5f5ef0568dfacd542e/.tmp/A/516acb04f2084f5ab2fb65855735bfac 2024-11-24T22:14:52,844 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/cdb3383a4019da5f5ef0568dfacd542e/.tmp/B/c89391f134b744d3b8c1a8421e8cf037 is 50, key is test_row_0/B:col10/1732486492687/Put/seqid=0 2024-11-24T22:14:52,854 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45929 is added to blk_1073742085_1261 (size=12001) 2024-11-24T22:14:52,854 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=96 (bloomFilter=true), to=hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/cdb3383a4019da5f5ef0568dfacd542e/.tmp/B/c89391f134b744d3b8c1a8421e8cf037 2024-11-24T22:14:52,863 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/cdb3383a4019da5f5ef0568dfacd542e/.tmp/C/3b88cd9780714be0ba6d5f570030a441 is 50, key is test_row_0/C:col10/1732486492687/Put/seqid=0 2024-11-24T22:14:52,867 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45929 is added to blk_1073742086_1262 (size=12001) 2024-11-24T22:14:52,868 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=96 (bloomFilter=true), to=hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/cdb3383a4019da5f5ef0568dfacd542e/.tmp/C/3b88cd9780714be0ba6d5f570030a441 2024-11-24T22:14:52,872 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/cdb3383a4019da5f5ef0568dfacd542e/.tmp/A/516acb04f2084f5ab2fb65855735bfac as hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/cdb3383a4019da5f5ef0568dfacd542e/A/516acb04f2084f5ab2fb65855735bfac 2024-11-24T22:14:52,876 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/cdb3383a4019da5f5ef0568dfacd542e/A/516acb04f2084f5ab2fb65855735bfac, entries=200, sequenceid=96, filesize=14.0 K 2024-11-24T22:14:52,878 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/cdb3383a4019da5f5ef0568dfacd542e/.tmp/B/c89391f134b744d3b8c1a8421e8cf037 as hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/cdb3383a4019da5f5ef0568dfacd542e/B/c89391f134b744d3b8c1a8421e8cf037 2024-11-24T22:14:52,883 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/cdb3383a4019da5f5ef0568dfacd542e/B/c89391f134b744d3b8c1a8421e8cf037, entries=150, sequenceid=96, filesize=11.7 K 2024-11-24T22:14:52,883 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/cdb3383a4019da5f5ef0568dfacd542e/.tmp/C/3b88cd9780714be0ba6d5f570030a441 as hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/cdb3383a4019da5f5ef0568dfacd542e/C/3b88cd9780714be0ba6d5f570030a441 2024-11-24T22:14:52,889 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/cdb3383a4019da5f5ef0568dfacd542e/C/3b88cd9780714be0ba6d5f570030a441, entries=150, sequenceid=96, filesize=11.7 K 2024-11-24T22:14:52,890 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~67.09 KB/68700, heapSize ~199.92 KB/204720, currentSize=80.51 KB/82440 for cdb3383a4019da5f5ef0568dfacd542e in 75ms, sequenceid=96, compaction requested=true 2024-11-24T22:14:52,890 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for cdb3383a4019da5f5ef0568dfacd542e: 2024-11-24T22:14:52,890 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store cdb3383a4019da5f5ef0568dfacd542e:A, priority=-2147483648, current under compaction store size is 1 2024-11-24T22:14:52,890 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-24T22:14:52,890 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store cdb3383a4019da5f5ef0568dfacd542e:B, priority=-2147483648, current under compaction store size is 2 2024-11-24T22:14:52,890 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-24T22:14:52,890 DEBUG [RS:0;bd6592c37bb2:40087-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-24T22:14:52,890 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store cdb3383a4019da5f5ef0568dfacd542e:C, priority=-2147483648, current under compaction store size is 3 2024-11-24T22:14:52,890 DEBUG [RS:0;bd6592c37bb2:40087-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-24T22:14:52,890 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-24T22:14:52,892 DEBUG [RS:0;bd6592c37bb2:40087-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 38446 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-24T22:14:52,892 DEBUG [RS:0;bd6592c37bb2:40087-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36106 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-24T22:14:52,892 DEBUG [RS:0;bd6592c37bb2:40087-longCompactions-0 {}] regionserver.HStore(1541): cdb3383a4019da5f5ef0568dfacd542e/B is initiating minor compaction (all files) 2024-11-24T22:14:52,892 DEBUG [RS:0;bd6592c37bb2:40087-shortCompactions-0 {}] regionserver.HStore(1541): cdb3383a4019da5f5ef0568dfacd542e/A is initiating minor compaction (all files) 2024-11-24T22:14:52,892 INFO [RS:0;bd6592c37bb2:40087-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of cdb3383a4019da5f5ef0568dfacd542e/B in TestAcidGuarantees,,1732486488674.cdb3383a4019da5f5ef0568dfacd542e. 2024-11-24T22:14:52,892 INFO [RS:0;bd6592c37bb2:40087-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of cdb3383a4019da5f5ef0568dfacd542e/A in TestAcidGuarantees,,1732486488674.cdb3383a4019da5f5ef0568dfacd542e. 2024-11-24T22:14:52,893 INFO [RS:0;bd6592c37bb2:40087-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/cdb3383a4019da5f5ef0568dfacd542e/A/7f34a3a979ba46fb954a392c6945ce2f, hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/cdb3383a4019da5f5ef0568dfacd542e/A/aa7e9dc47e4f4e2ebe297d6e390500bd, hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/cdb3383a4019da5f5ef0568dfacd542e/A/516acb04f2084f5ab2fb65855735bfac] into tmpdir=hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/cdb3383a4019da5f5ef0568dfacd542e/.tmp, totalSize=37.5 K 2024-11-24T22:14:52,893 INFO [RS:0;bd6592c37bb2:40087-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/cdb3383a4019da5f5ef0568dfacd542e/B/e5b76e9a2c6b4facac53b583784c0fb7, hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/cdb3383a4019da5f5ef0568dfacd542e/B/3068e6ac5958446782debae8e550dc98, hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/cdb3383a4019da5f5ef0568dfacd542e/B/c89391f134b744d3b8c1a8421e8cf037] into tmpdir=hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/cdb3383a4019da5f5ef0568dfacd542e/.tmp, totalSize=35.3 K 2024-11-24T22:14:52,893 DEBUG [RS:0;bd6592c37bb2:40087-longCompactions-0 {}] compactions.Compactor(225): Compacting e5b76e9a2c6b4facac53b583784c0fb7, keycount=150, bloomtype=ROW, size=11.8 K, encoding=NONE, compression=NONE, seqNum=60, earliestPutTs=1732486491249 2024-11-24T22:14:52,893 DEBUG [RS:0;bd6592c37bb2:40087-shortCompactions-0 {}] compactions.Compactor(225): Compacting 7f34a3a979ba46fb954a392c6945ce2f, keycount=150, bloomtype=ROW, size=11.8 K, encoding=NONE, compression=NONE, seqNum=60, earliestPutTs=1732486491249 2024-11-24T22:14:52,893 DEBUG [RS:0;bd6592c37bb2:40087-longCompactions-0 {}] compactions.Compactor(225): Compacting 3068e6ac5958446782debae8e550dc98, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=80, earliestPutTs=1732486491473 2024-11-24T22:14:52,893 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40087 {}] regionserver.HRegion(8855): Flush requested on cdb3383a4019da5f5ef0568dfacd542e 2024-11-24T22:14:52,894 DEBUG [RS:0;bd6592c37bb2:40087-shortCompactions-0 {}] compactions.Compactor(225): Compacting aa7e9dc47e4f4e2ebe297d6e390500bd, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=80, earliestPutTs=1732486491473 2024-11-24T22:14:52,894 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing cdb3383a4019da5f5ef0568dfacd542e 3/3 column families, dataSize=87.22 KB heapSize=259.73 KB 2024-11-24T22:14:52,894 DEBUG [RS:0;bd6592c37bb2:40087-shortCompactions-0 {}] compactions.Compactor(225): Compacting 516acb04f2084f5ab2fb65855735bfac, keycount=200, bloomtype=ROW, size=14.0 K, encoding=NONE, compression=NONE, seqNum=96, earliestPutTs=1732486492664 2024-11-24T22:14:52,894 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK cdb3383a4019da5f5ef0568dfacd542e, store=A 2024-11-24T22:14:52,894 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-24T22:14:52,894 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK cdb3383a4019da5f5ef0568dfacd542e, store=B 2024-11-24T22:14:52,894 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-24T22:14:52,894 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK cdb3383a4019da5f5ef0568dfacd542e, store=C 2024-11-24T22:14:52,894 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-24T22:14:52,894 DEBUG [RS:0;bd6592c37bb2:40087-longCompactions-0 {}] compactions.Compactor(225): Compacting c89391f134b744d3b8c1a8421e8cf037, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=96, earliestPutTs=1732486492664 2024-11-24T22:14:52,900 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/cdb3383a4019da5f5ef0568dfacd542e/.tmp/A/20eee9fcfcec4fceab01ad537875e97e is 50, key is test_row_0/A:col10/1732486492891/Put/seqid=0 2024-11-24T22:14:52,907 INFO [RS:0;bd6592c37bb2:40087-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): cdb3383a4019da5f5ef0568dfacd542e#A#compaction#214 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-24T22:14:52,908 DEBUG [RS:0;bd6592c37bb2:40087-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/cdb3383a4019da5f5ef0568dfacd542e/.tmp/A/97b091d10a0d4c3e9b134e1fc633706f is 50, key is test_row_0/A:col10/1732486492687/Put/seqid=0 2024-11-24T22:14:52,918 INFO [RS:0;bd6592c37bb2:40087-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): cdb3383a4019da5f5ef0568dfacd542e#B#compaction#215 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-24T22:14:52,918 DEBUG [RS:0;bd6592c37bb2:40087-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/cdb3383a4019da5f5ef0568dfacd542e/.tmp/B/b68df242e91f4b989faa62390bff6f0d is 50, key is test_row_0/B:col10/1732486492687/Put/seqid=0 2024-11-24T22:14:52,919 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45929 is added to blk_1073742088_1264 (size=12207) 2024-11-24T22:14:52,922 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45929 is added to blk_1073742087_1263 (size=12001) 2024-11-24T22:14:52,923 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=29.07 KB at sequenceid=112 (bloomFilter=true), to=hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/cdb3383a4019da5f5ef0568dfacd542e/.tmp/A/20eee9fcfcec4fceab01ad537875e97e 2024-11-24T22:14:52,934 DEBUG [RS:0;bd6592c37bb2:40087-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/cdb3383a4019da5f5ef0568dfacd542e/.tmp/A/97b091d10a0d4c3e9b134e1fc633706f as hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/cdb3383a4019da5f5ef0568dfacd542e/A/97b091d10a0d4c3e9b134e1fc633706f 2024-11-24T22:14:52,940 INFO [RS:0;bd6592c37bb2:40087-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in cdb3383a4019da5f5ef0568dfacd542e/A of cdb3383a4019da5f5ef0568dfacd542e into 97b091d10a0d4c3e9b134e1fc633706f(size=11.9 K), total size for store is 11.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-24T22:14:52,940 DEBUG [RS:0;bd6592c37bb2:40087-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for cdb3383a4019da5f5ef0568dfacd542e: 2024-11-24T22:14:52,940 INFO [RS:0;bd6592c37bb2:40087-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732486488674.cdb3383a4019da5f5ef0568dfacd542e., storeName=cdb3383a4019da5f5ef0568dfacd542e/A, priority=13, startTime=1732486492890; duration=0sec 2024-11-24T22:14:52,940 DEBUG [RS:0;bd6592c37bb2:40087-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-24T22:14:52,940 DEBUG [RS:0;bd6592c37bb2:40087-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: cdb3383a4019da5f5ef0568dfacd542e:A 2024-11-24T22:14:52,940 DEBUG [RS:0;bd6592c37bb2:40087-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-24T22:14:52,942 DEBUG [RS:0;bd6592c37bb2:40087-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36106 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-24T22:14:52,942 DEBUG [RS:0;bd6592c37bb2:40087-shortCompactions-0 {}] regionserver.HStore(1541): cdb3383a4019da5f5ef0568dfacd542e/C is initiating minor compaction (all files) 2024-11-24T22:14:52,942 INFO [RS:0;bd6592c37bb2:40087-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of cdb3383a4019da5f5ef0568dfacd542e/C in TestAcidGuarantees,,1732486488674.cdb3383a4019da5f5ef0568dfacd542e. 2024-11-24T22:14:52,942 INFO [RS:0;bd6592c37bb2:40087-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/cdb3383a4019da5f5ef0568dfacd542e/C/73ebb44821fc486aa70667b86beeabc6, hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/cdb3383a4019da5f5ef0568dfacd542e/C/54172941587641b39b3076d159e66605, hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/cdb3383a4019da5f5ef0568dfacd542e/C/3b88cd9780714be0ba6d5f570030a441] into tmpdir=hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/cdb3383a4019da5f5ef0568dfacd542e/.tmp, totalSize=35.3 K 2024-11-24T22:14:52,942 DEBUG [RS:0;bd6592c37bb2:40087-shortCompactions-0 {}] compactions.Compactor(225): Compacting 73ebb44821fc486aa70667b86beeabc6, keycount=150, bloomtype=ROW, size=11.8 K, encoding=NONE, compression=NONE, seqNum=60, earliestPutTs=1732486491249 2024-11-24T22:14:52,943 DEBUG [RS:0;bd6592c37bb2:40087-shortCompactions-0 {}] compactions.Compactor(225): Compacting 54172941587641b39b3076d159e66605, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=80, earliestPutTs=1732486491473 2024-11-24T22:14:52,943 DEBUG [RS:0;bd6592c37bb2:40087-shortCompactions-0 {}] compactions.Compactor(225): Compacting 3b88cd9780714be0ba6d5f570030a441, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=96, earliestPutTs=1732486492664 2024-11-24T22:14:52,947 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/cdb3383a4019da5f5ef0568dfacd542e/.tmp/B/65d304270e9042d58934bac821362e54 is 50, key is test_row_0/B:col10/1732486492891/Put/seqid=0 2024-11-24T22:14:52,950 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45929 is added to blk_1073742089_1265 (size=12207) 2024-11-24T22:14:52,957 DEBUG [RS:0;bd6592c37bb2:40087-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/cdb3383a4019da5f5ef0568dfacd542e/.tmp/B/b68df242e91f4b989faa62390bff6f0d as hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/cdb3383a4019da5f5ef0568dfacd542e/B/b68df242e91f4b989faa62390bff6f0d 2024-11-24T22:14:52,962 INFO [RS:0;bd6592c37bb2:40087-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in cdb3383a4019da5f5ef0568dfacd542e/B of cdb3383a4019da5f5ef0568dfacd542e into b68df242e91f4b989faa62390bff6f0d(size=11.9 K), total size for store is 11.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-24T22:14:52,962 DEBUG [RS:0;bd6592c37bb2:40087-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for cdb3383a4019da5f5ef0568dfacd542e: 2024-11-24T22:14:52,962 INFO [RS:0;bd6592c37bb2:40087-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732486488674.cdb3383a4019da5f5ef0568dfacd542e., storeName=cdb3383a4019da5f5ef0568dfacd542e/B, priority=13, startTime=1732486492890; duration=0sec 2024-11-24T22:14:52,963 DEBUG [RS:0;bd6592c37bb2:40087-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-24T22:14:52,963 DEBUG [RS:0;bd6592c37bb2:40087-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: cdb3383a4019da5f5ef0568dfacd542e:B 2024-11-24T22:14:52,966 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45929 is added to blk_1073742090_1266 (size=12001) 2024-11-24T22:14:52,967 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=29.07 KB at sequenceid=112 (bloomFilter=true), to=hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/cdb3383a4019da5f5ef0568dfacd542e/.tmp/B/65d304270e9042d58934bac821362e54 2024-11-24T22:14:52,968 INFO [RS:0;bd6592c37bb2:40087-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): cdb3383a4019da5f5ef0568dfacd542e#C#compaction#217 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-24T22:14:52,969 DEBUG [RS:0;bd6592c37bb2:40087-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/cdb3383a4019da5f5ef0568dfacd542e/.tmp/C/48dcc83ad5644812ba027d775c3a45f7 is 50, key is test_row_0/C:col10/1732486492687/Put/seqid=0 2024-11-24T22:14:52,978 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/cdb3383a4019da5f5ef0568dfacd542e/.tmp/C/42c1b6cf23734fef8a3529416a6735e3 is 50, key is test_row_0/C:col10/1732486492891/Put/seqid=0 2024-11-24T22:14:52,980 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45929 is added to blk_1073742091_1267 (size=12207) 2024-11-24T22:14:52,983 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45929 is added to blk_1073742092_1268 (size=12001) 2024-11-24T22:14:52,992 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40087 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cdb3383a4019da5f5ef0568dfacd542e, server=bd6592c37bb2,40087,1732486423713 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-24T22:14:52,992 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40087 {}] ipc.CallRunner(138): callId: 75 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40834 deadline: 1732486552988, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cdb3383a4019da5f5ef0568dfacd542e, server=bd6592c37bb2,40087,1732486423713 2024-11-24T22:14:52,993 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732486488674.cdb3383a4019da5f5ef0568dfacd542e., hostname=bd6592c37bb2,40087,1732486423713, seqNum=2 , the old value is region=TestAcidGuarantees,,1732486488674.cdb3383a4019da5f5ef0568dfacd542e., hostname=bd6592c37bb2,40087,1732486423713, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cdb3383a4019da5f5ef0568dfacd542e, server=bd6592c37bb2,40087,1732486423713 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T22:14:52,993 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732486488674.cdb3383a4019da5f5ef0568dfacd542e., hostname=bd6592c37bb2,40087,1732486423713, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cdb3383a4019da5f5ef0568dfacd542e, server=bd6592c37bb2,40087,1732486423713 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T22:14:52,993 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732486488674.cdb3383a4019da5f5ef0568dfacd542e., hostname=bd6592c37bb2,40087,1732486423713, seqNum=2 because the exception is null or not the one we care about 2024-11-24T22:14:53,104 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40087 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cdb3383a4019da5f5ef0568dfacd542e, server=bd6592c37bb2,40087,1732486423713 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-24T22:14:53,104 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40087 {}] ipc.CallRunner(138): callId: 76 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40834 deadline: 1732486553101, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cdb3383a4019da5f5ef0568dfacd542e, server=bd6592c37bb2,40087,1732486423713 2024-11-24T22:14:53,104 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732486488674.cdb3383a4019da5f5ef0568dfacd542e., hostname=bd6592c37bb2,40087,1732486423713, seqNum=2 , the old value is region=TestAcidGuarantees,,1732486488674.cdb3383a4019da5f5ef0568dfacd542e., hostname=bd6592c37bb2,40087,1732486423713, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cdb3383a4019da5f5ef0568dfacd542e, server=bd6592c37bb2,40087,1732486423713 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T22:14:53,104 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732486488674.cdb3383a4019da5f5ef0568dfacd542e., hostname=bd6592c37bb2,40087,1732486423713, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cdb3383a4019da5f5ef0568dfacd542e, server=bd6592c37bb2,40087,1732486423713 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T22:14:53,104 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732486488674.cdb3383a4019da5f5ef0568dfacd542e., hostname=bd6592c37bb2,40087,1732486423713, seqNum=2 because the exception is null or not the one we care about 2024-11-24T22:14:53,124 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40087 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cdb3383a4019da5f5ef0568dfacd542e, server=bd6592c37bb2,40087,1732486423713 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-24T22:14:53,124 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40087 {}] ipc.CallRunner(138): callId: 16 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40772 deadline: 1732486553121, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cdb3383a4019da5f5ef0568dfacd542e, server=bd6592c37bb2,40087,1732486423713 2024-11-24T22:14:53,124 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40087 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cdb3383a4019da5f5ef0568dfacd542e, server=bd6592c37bb2,40087,1732486423713 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-24T22:14:53,125 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40087 {}] ipc.CallRunner(138): callId: 15 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40816 deadline: 1732486553121, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cdb3383a4019da5f5ef0568dfacd542e, server=bd6592c37bb2,40087,1732486423713 2024-11-24T22:14:53,125 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40087 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cdb3383a4019da5f5ef0568dfacd542e, server=bd6592c37bb2,40087,1732486423713 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-24T22:14:53,125 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40087 {}] ipc.CallRunner(138): callId: 15 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40842 deadline: 1732486553121, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cdb3383a4019da5f5ef0568dfacd542e, server=bd6592c37bb2,40087,1732486423713 2024-11-24T22:14:53,125 WARN [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRpcRetryingCaller(177): Call to bd6592c37bb2,40087,1732486423713 for 'test_row_2' in cdb3383a4019da5f5ef0568dfacd542e of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2169 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cdb3383a4019da5f5ef0568dfacd542e, server=bd6592c37bb2,40087,1732486423713 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor44.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cdb3383a4019da5f5ef0568dfacd542e, server=bd6592c37bb2,40087,1732486423713 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-24T22:14:53,125 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40087 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cdb3383a4019da5f5ef0568dfacd542e, server=bd6592c37bb2,40087,1732486423713 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-24T22:14:53,125 WARN [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRpcRetryingCaller(177): Call to bd6592c37bb2,40087,1732486423713 for 'test_row_2' in cdb3383a4019da5f5ef0568dfacd542e of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2167 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cdb3383a4019da5f5ef0568dfacd542e, server=bd6592c37bb2,40087,1732486423713 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor44.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cdb3383a4019da5f5ef0568dfacd542e, server=bd6592c37bb2,40087,1732486423713 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-24T22:14:53,125 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40087 {}] ipc.CallRunner(138): callId: 15 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40806 deadline: 1732486553121, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cdb3383a4019da5f5ef0568dfacd542e, server=bd6592c37bb2,40087,1732486423713 2024-11-24T22:14:53,125 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732486488674.cdb3383a4019da5f5ef0568dfacd542e., hostname=bd6592c37bb2,40087,1732486423713, seqNum=2 , the old value is region=TestAcidGuarantees,,1732486488674.cdb3383a4019da5f5ef0568dfacd542e., hostname=bd6592c37bb2,40087,1732486423713, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cdb3383a4019da5f5ef0568dfacd542e, server=bd6592c37bb2,40087,1732486423713 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T22:14:53,125 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732486488674.cdb3383a4019da5f5ef0568dfacd542e., hostname=bd6592c37bb2,40087,1732486423713, seqNum=2 , the old value is region=TestAcidGuarantees,,1732486488674.cdb3383a4019da5f5ef0568dfacd542e., hostname=bd6592c37bb2,40087,1732486423713, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cdb3383a4019da5f5ef0568dfacd542e, server=bd6592c37bb2,40087,1732486423713 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T22:14:53,126 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732486488674.cdb3383a4019da5f5ef0568dfacd542e., hostname=bd6592c37bb2,40087,1732486423713, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cdb3383a4019da5f5ef0568dfacd542e, server=bd6592c37bb2,40087,1732486423713 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T22:14:53,126 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732486488674.cdb3383a4019da5f5ef0568dfacd542e., hostname=bd6592c37bb2,40087,1732486423713, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cdb3383a4019da5f5ef0568dfacd542e, server=bd6592c37bb2,40087,1732486423713 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T22:14:53,126 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732486488674.cdb3383a4019da5f5ef0568dfacd542e., hostname=bd6592c37bb2,40087,1732486423713, seqNum=2 because the exception is null or not the one we care about 2024-11-24T22:14:53,126 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732486488674.cdb3383a4019da5f5ef0568dfacd542e., hostname=bd6592c37bb2,40087,1732486423713, seqNum=2 because the exception is null or not the one we care about 2024-11-24T22:14:53,125 WARN [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRpcRetryingCaller(177): Call to bd6592c37bb2,40087,1732486423713 for 'test_row_0' in cdb3383a4019da5f5ef0568dfacd542e of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2167 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cdb3383a4019da5f5ef0568dfacd542e, server=bd6592c37bb2,40087,1732486423713 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor44.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cdb3383a4019da5f5ef0568dfacd542e, server=bd6592c37bb2,40087,1732486423713 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-24T22:14:53,126 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732486488674.cdb3383a4019da5f5ef0568dfacd542e., hostname=bd6592c37bb2,40087,1732486423713, seqNum=2 , the old value is region=TestAcidGuarantees,,1732486488674.cdb3383a4019da5f5ef0568dfacd542e., hostname=bd6592c37bb2,40087,1732486423713, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cdb3383a4019da5f5ef0568dfacd542e, server=bd6592c37bb2,40087,1732486423713 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T22:14:53,126 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732486488674.cdb3383a4019da5f5ef0568dfacd542e., hostname=bd6592c37bb2,40087,1732486423713, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cdb3383a4019da5f5ef0568dfacd542e, server=bd6592c37bb2,40087,1732486423713 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T22:14:53,126 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732486488674.cdb3383a4019da5f5ef0568dfacd542e., hostname=bd6592c37bb2,40087,1732486423713, seqNum=2 because the exception is null or not the one we care about 2024-11-24T22:14:53,126 WARN [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRpcRetryingCaller(177): Call to bd6592c37bb2,40087,1732486423713 for 'test_row_2' in cdb3383a4019da5f5ef0568dfacd542e of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2167 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cdb3383a4019da5f5ef0568dfacd542e, server=bd6592c37bb2,40087,1732486423713 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor44.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cdb3383a4019da5f5ef0568dfacd542e, server=bd6592c37bb2,40087,1732486423713 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-24T22:14:53,126 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732486488674.cdb3383a4019da5f5ef0568dfacd542e., hostname=bd6592c37bb2,40087,1732486423713, seqNum=2 , the old value is region=TestAcidGuarantees,,1732486488674.cdb3383a4019da5f5ef0568dfacd542e., hostname=bd6592c37bb2,40087,1732486423713, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cdb3383a4019da5f5ef0568dfacd542e, server=bd6592c37bb2,40087,1732486423713 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T22:14:53,126 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732486488674.cdb3383a4019da5f5ef0568dfacd542e., hostname=bd6592c37bb2,40087,1732486423713, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cdb3383a4019da5f5ef0568dfacd542e, server=bd6592c37bb2,40087,1732486423713 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T22:14:53,127 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732486488674.cdb3383a4019da5f5ef0568dfacd542e., hostname=bd6592c37bb2,40087,1732486423713, seqNum=2 because the exception is null or not the one we care about 2024-11-24T22:14:53,315 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40087 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cdb3383a4019da5f5ef0568dfacd542e, server=bd6592c37bb2,40087,1732486423713 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-24T22:14:53,315 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40087 {}] ipc.CallRunner(138): callId: 77 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40834 deadline: 1732486553311, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cdb3383a4019da5f5ef0568dfacd542e, server=bd6592c37bb2,40087,1732486423713 2024-11-24T22:14:53,315 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732486488674.cdb3383a4019da5f5ef0568dfacd542e., hostname=bd6592c37bb2,40087,1732486423713, seqNum=2 , the old value is region=TestAcidGuarantees,,1732486488674.cdb3383a4019da5f5ef0568dfacd542e., hostname=bd6592c37bb2,40087,1732486423713, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cdb3383a4019da5f5ef0568dfacd542e, server=bd6592c37bb2,40087,1732486423713 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T22:14:53,315 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732486488674.cdb3383a4019da5f5ef0568dfacd542e., hostname=bd6592c37bb2,40087,1732486423713, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cdb3383a4019da5f5ef0568dfacd542e, server=bd6592c37bb2,40087,1732486423713 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T22:14:53,315 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732486488674.cdb3383a4019da5f5ef0568dfacd542e., hostname=bd6592c37bb2,40087,1732486423713, seqNum=2 because the exception is null or not the one we care about 2024-11-24T22:14:53,384 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=29.07 KB at sequenceid=112 (bloomFilter=true), to=hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/cdb3383a4019da5f5ef0568dfacd542e/.tmp/C/42c1b6cf23734fef8a3529416a6735e3 2024-11-24T22:14:53,387 DEBUG [RS:0;bd6592c37bb2:40087-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/cdb3383a4019da5f5ef0568dfacd542e/.tmp/C/48dcc83ad5644812ba027d775c3a45f7 as hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/cdb3383a4019da5f5ef0568dfacd542e/C/48dcc83ad5644812ba027d775c3a45f7 2024-11-24T22:14:53,388 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/cdb3383a4019da5f5ef0568dfacd542e/.tmp/A/20eee9fcfcec4fceab01ad537875e97e as hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/cdb3383a4019da5f5ef0568dfacd542e/A/20eee9fcfcec4fceab01ad537875e97e 2024-11-24T22:14:53,391 INFO [RS:0;bd6592c37bb2:40087-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in cdb3383a4019da5f5ef0568dfacd542e/C of cdb3383a4019da5f5ef0568dfacd542e into 48dcc83ad5644812ba027d775c3a45f7(size=11.9 K), total size for store is 11.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-24T22:14:53,391 DEBUG [RS:0;bd6592c37bb2:40087-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for cdb3383a4019da5f5ef0568dfacd542e: 2024-11-24T22:14:53,391 INFO [RS:0;bd6592c37bb2:40087-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732486488674.cdb3383a4019da5f5ef0568dfacd542e., storeName=cdb3383a4019da5f5ef0568dfacd542e/C, priority=13, startTime=1732486492890; duration=0sec 2024-11-24T22:14:53,391 DEBUG [RS:0;bd6592c37bb2:40087-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-24T22:14:53,392 DEBUG [RS:0;bd6592c37bb2:40087-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: cdb3383a4019da5f5ef0568dfacd542e:C 2024-11-24T22:14:53,392 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/cdb3383a4019da5f5ef0568dfacd542e/A/20eee9fcfcec4fceab01ad537875e97e, entries=150, sequenceid=112, filesize=11.7 K 2024-11-24T22:14:53,393 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/cdb3383a4019da5f5ef0568dfacd542e/.tmp/B/65d304270e9042d58934bac821362e54 as hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/cdb3383a4019da5f5ef0568dfacd542e/B/65d304270e9042d58934bac821362e54 2024-11-24T22:14:53,397 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/cdb3383a4019da5f5ef0568dfacd542e/B/65d304270e9042d58934bac821362e54, entries=150, sequenceid=112, filesize=11.7 K 2024-11-24T22:14:53,398 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/cdb3383a4019da5f5ef0568dfacd542e/.tmp/C/42c1b6cf23734fef8a3529416a6735e3 as hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/cdb3383a4019da5f5ef0568dfacd542e/C/42c1b6cf23734fef8a3529416a6735e3 2024-11-24T22:14:53,402 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/cdb3383a4019da5f5ef0568dfacd542e/C/42c1b6cf23734fef8a3529416a6735e3, entries=150, sequenceid=112, filesize=11.7 K 2024-11-24T22:14:53,402 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~87.22 KB/89310, heapSize ~259.69 KB/265920, currentSize=87.22 KB/89310 for cdb3383a4019da5f5ef0568dfacd542e in 508ms, sequenceid=112, compaction requested=false 2024-11-24T22:14:53,402 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for cdb3383a4019da5f5ef0568dfacd542e: 2024-11-24T22:14:53,623 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40087 {}] regionserver.HRegion(8855): Flush requested on cdb3383a4019da5f5ef0568dfacd542e 2024-11-24T22:14:53,623 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing cdb3383a4019da5f5ef0568dfacd542e 3/3 column families, dataSize=93.93 KB heapSize=279.66 KB 2024-11-24T22:14:53,624 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK cdb3383a4019da5f5ef0568dfacd542e, store=A 2024-11-24T22:14:53,624 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-24T22:14:53,624 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK cdb3383a4019da5f5ef0568dfacd542e, store=B 2024-11-24T22:14:53,624 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-24T22:14:53,624 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK cdb3383a4019da5f5ef0568dfacd542e, store=C 2024-11-24T22:14:53,624 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-24T22:14:53,629 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/cdb3383a4019da5f5ef0568dfacd542e/.tmp/A/fa66f306f0f04c7d8adb9bd8978e52a5 is 50, key is test_row_0/A:col10/1732486493622/Put/seqid=0 2024-11-24T22:14:53,633 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45929 is added to blk_1073742093_1269 (size=14441) 2024-11-24T22:14:53,634 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=31.31 KB at sequenceid=132 (bloomFilter=true), to=hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/cdb3383a4019da5f5ef0568dfacd542e/.tmp/A/fa66f306f0f04c7d8adb9bd8978e52a5 2024-11-24T22:14:53,642 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/cdb3383a4019da5f5ef0568dfacd542e/.tmp/B/c6d4b3ff05d942349fdf2d2a9a078c6c is 50, key is test_row_0/B:col10/1732486493622/Put/seqid=0 2024-11-24T22:14:53,651 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37701 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=62 2024-11-24T22:14:53,652 INFO [RPCClient-NioEventLoopGroup-4-6 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-24T22:14:53,652 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37701 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-24T22:14:53,653 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37701 {}] procedure2.ProcedureExecutor(1139): Stored pid=64, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=64, table=TestAcidGuarantees 2024-11-24T22:14:53,654 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37701 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=64 2024-11-24T22:14:53,654 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=64, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=64, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-24T22:14:53,655 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=64, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=64, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-24T22:14:53,655 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=65, ppid=64, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-24T22:14:53,666 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45929 is added to blk_1073742094_1270 (size=12051) 2024-11-24T22:14:53,691 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40087 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cdb3383a4019da5f5ef0568dfacd542e, server=bd6592c37bb2,40087,1732486423713 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-24T22:14:53,691 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40087 {}] ipc.CallRunner(138): callId: 91 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40834 deadline: 1732486553687, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cdb3383a4019da5f5ef0568dfacd542e, server=bd6592c37bb2,40087,1732486423713 2024-11-24T22:14:53,692 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732486488674.cdb3383a4019da5f5ef0568dfacd542e., hostname=bd6592c37bb2,40087,1732486423713, seqNum=2 , the old value is region=TestAcidGuarantees,,1732486488674.cdb3383a4019da5f5ef0568dfacd542e., hostname=bd6592c37bb2,40087,1732486423713, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cdb3383a4019da5f5ef0568dfacd542e, server=bd6592c37bb2,40087,1732486423713 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T22:14:53,692 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732486488674.cdb3383a4019da5f5ef0568dfacd542e., hostname=bd6592c37bb2,40087,1732486423713, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cdb3383a4019da5f5ef0568dfacd542e, server=bd6592c37bb2,40087,1732486423713 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T22:14:53,692 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732486488674.cdb3383a4019da5f5ef0568dfacd542e., hostname=bd6592c37bb2,40087,1732486423713, seqNum=2 because the exception is null or not the one we care about 2024-11-24T22:14:53,747 WARN [HBase-Metrics2-1 {}] impl.MetricsConfig(138): Cannot locate configuration: tried hadoop-metrics2-hbase.properties,hadoop-metrics2.properties 2024-11-24T22:14:53,761 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37701 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=64 2024-11-24T22:14:53,803 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40087 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cdb3383a4019da5f5ef0568dfacd542e, server=bd6592c37bb2,40087,1732486423713 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-24T22:14:53,804 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40087 {}] ipc.CallRunner(138): callId: 92 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40834 deadline: 1732486553801, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cdb3383a4019da5f5ef0568dfacd542e, server=bd6592c37bb2,40087,1732486423713 2024-11-24T22:14:53,804 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732486488674.cdb3383a4019da5f5ef0568dfacd542e., hostname=bd6592c37bb2,40087,1732486423713, seqNum=2 , the old value is region=TestAcidGuarantees,,1732486488674.cdb3383a4019da5f5ef0568dfacd542e., hostname=bd6592c37bb2,40087,1732486423713, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cdb3383a4019da5f5ef0568dfacd542e, server=bd6592c37bb2,40087,1732486423713 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T22:14:53,804 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732486488674.cdb3383a4019da5f5ef0568dfacd542e., hostname=bd6592c37bb2,40087,1732486423713, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cdb3383a4019da5f5ef0568dfacd542e, server=bd6592c37bb2,40087,1732486423713 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T22:14:53,804 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732486488674.cdb3383a4019da5f5ef0568dfacd542e., hostname=bd6592c37bb2,40087,1732486423713, seqNum=2 because the exception is null or not the one we care about 2024-11-24T22:14:53,807 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=40087 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=65 2024-11-24T22:14:53,807 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bd6592c37bb2:0-1 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732486488674.cdb3383a4019da5f5ef0568dfacd542e. 2024-11-24T22:14:53,807 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bd6592c37bb2:0-1 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732486488674.cdb3383a4019da5f5ef0568dfacd542e. as already flushing 2024-11-24T22:14:53,807 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bd6592c37bb2:0-1 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732486488674.cdb3383a4019da5f5ef0568dfacd542e. 2024-11-24T22:14:53,807 ERROR [RS_FLUSH_OPERATIONS-regionserver/bd6592c37bb2:0-1 {event_type=RS_FLUSH_REGIONS, pid=65}] handler.RSProcedureHandler(58): pid=65 java.io.IOException: Unable to complete flush {ENCODED => cdb3383a4019da5f5ef0568dfacd542e, NAME => 'TestAcidGuarantees,,1732486488674.cdb3383a4019da5f5ef0568dfacd542e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T22:14:53,807 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bd6592c37bb2:0-1 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=65 java.io.IOException: Unable to complete flush {ENCODED => cdb3383a4019da5f5ef0568dfacd542e, NAME => 'TestAcidGuarantees,,1732486488674.cdb3383a4019da5f5ef0568dfacd542e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T22:14:53,808 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37701 {}] master.HMaster(4177): Remote procedure failed, pid=65 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => cdb3383a4019da5f5ef0568dfacd542e, NAME => 'TestAcidGuarantees,,1732486488674.cdb3383a4019da5f5ef0568dfacd542e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => cdb3383a4019da5f5ef0568dfacd542e, NAME => 'TestAcidGuarantees,,1732486488674.cdb3383a4019da5f5ef0568dfacd542e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T22:14:53,960 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=40087 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=65 2024-11-24T22:14:53,960 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bd6592c37bb2:0-2 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732486488674.cdb3383a4019da5f5ef0568dfacd542e. 2024-11-24T22:14:53,960 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bd6592c37bb2:0-2 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732486488674.cdb3383a4019da5f5ef0568dfacd542e. as already flushing 2024-11-24T22:14:53,960 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bd6592c37bb2:0-2 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732486488674.cdb3383a4019da5f5ef0568dfacd542e. 2024-11-24T22:14:53,961 ERROR [RS_FLUSH_OPERATIONS-regionserver/bd6592c37bb2:0-2 {event_type=RS_FLUSH_REGIONS, pid=65}] handler.RSProcedureHandler(58): pid=65 java.io.IOException: Unable to complete flush {ENCODED => cdb3383a4019da5f5ef0568dfacd542e, NAME => 'TestAcidGuarantees,,1732486488674.cdb3383a4019da5f5ef0568dfacd542e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T22:14:53,961 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bd6592c37bb2:0-2 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=65 java.io.IOException: Unable to complete flush {ENCODED => cdb3383a4019da5f5ef0568dfacd542e, NAME => 'TestAcidGuarantees,,1732486488674.cdb3383a4019da5f5ef0568dfacd542e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T22:14:53,961 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37701 {}] master.HMaster(4177): Remote procedure failed, pid=65 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => cdb3383a4019da5f5ef0568dfacd542e, NAME => 'TestAcidGuarantees,,1732486488674.cdb3383a4019da5f5ef0568dfacd542e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => cdb3383a4019da5f5ef0568dfacd542e, NAME => 'TestAcidGuarantees,,1732486488674.cdb3383a4019da5f5ef0568dfacd542e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T22:14:53,971 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37701 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=64 2024-11-24T22:14:54,012 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40087 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cdb3383a4019da5f5ef0568dfacd542e, server=bd6592c37bb2,40087,1732486423713 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-24T22:14:54,012 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40087 {}] ipc.CallRunner(138): callId: 93 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40834 deadline: 1732486554011, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cdb3383a4019da5f5ef0568dfacd542e, server=bd6592c37bb2,40087,1732486423713 2024-11-24T22:14:54,013 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732486488674.cdb3383a4019da5f5ef0568dfacd542e., hostname=bd6592c37bb2,40087,1732486423713, seqNum=2 , the old value is region=TestAcidGuarantees,,1732486488674.cdb3383a4019da5f5ef0568dfacd542e., hostname=bd6592c37bb2,40087,1732486423713, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cdb3383a4019da5f5ef0568dfacd542e, server=bd6592c37bb2,40087,1732486423713 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T22:14:54,013 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732486488674.cdb3383a4019da5f5ef0568dfacd542e., hostname=bd6592c37bb2,40087,1732486423713, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cdb3383a4019da5f5ef0568dfacd542e, server=bd6592c37bb2,40087,1732486423713 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T22:14:54,013 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732486488674.cdb3383a4019da5f5ef0568dfacd542e., hostname=bd6592c37bb2,40087,1732486423713, seqNum=2 because the exception is null or not the one we care about 2024-11-24T22:14:54,066 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=31.31 KB at sequenceid=132 (bloomFilter=true), to=hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/cdb3383a4019da5f5ef0568dfacd542e/.tmp/B/c6d4b3ff05d942349fdf2d2a9a078c6c 2024-11-24T22:14:54,074 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/cdb3383a4019da5f5ef0568dfacd542e/.tmp/C/112218e9c5d848ba91829de0208e34b2 is 50, key is test_row_0/C:col10/1732486493622/Put/seqid=0 2024-11-24T22:14:54,078 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45929 is added to blk_1073742095_1271 (size=12051) 2024-11-24T22:14:54,113 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=40087 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=65 2024-11-24T22:14:54,114 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bd6592c37bb2:0-0 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732486488674.cdb3383a4019da5f5ef0568dfacd542e. 2024-11-24T22:14:54,114 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bd6592c37bb2:0-0 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732486488674.cdb3383a4019da5f5ef0568dfacd542e. as already flushing 2024-11-24T22:14:54,114 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bd6592c37bb2:0-0 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732486488674.cdb3383a4019da5f5ef0568dfacd542e. 2024-11-24T22:14:54,114 ERROR [RS_FLUSH_OPERATIONS-regionserver/bd6592c37bb2:0-0 {event_type=RS_FLUSH_REGIONS, pid=65}] handler.RSProcedureHandler(58): pid=65 java.io.IOException: Unable to complete flush {ENCODED => cdb3383a4019da5f5ef0568dfacd542e, NAME => 'TestAcidGuarantees,,1732486488674.cdb3383a4019da5f5ef0568dfacd542e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T22:14:54,114 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bd6592c37bb2:0-0 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=65 java.io.IOException: Unable to complete flush {ENCODED => cdb3383a4019da5f5ef0568dfacd542e, NAME => 'TestAcidGuarantees,,1732486488674.cdb3383a4019da5f5ef0568dfacd542e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T22:14:54,114 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37701 {}] master.HMaster(4177): Remote procedure failed, pid=65 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => cdb3383a4019da5f5ef0568dfacd542e, NAME => 'TestAcidGuarantees,,1732486488674.cdb3383a4019da5f5ef0568dfacd542e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => cdb3383a4019da5f5ef0568dfacd542e, NAME => 'TestAcidGuarantees,,1732486488674.cdb3383a4019da5f5ef0568dfacd542e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T22:14:54,267 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=40087 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=65 2024-11-24T22:14:54,267 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bd6592c37bb2:0-1 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732486488674.cdb3383a4019da5f5ef0568dfacd542e. 2024-11-24T22:14:54,267 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bd6592c37bb2:0-1 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732486488674.cdb3383a4019da5f5ef0568dfacd542e. as already flushing 2024-11-24T22:14:54,267 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bd6592c37bb2:0-1 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732486488674.cdb3383a4019da5f5ef0568dfacd542e. 2024-11-24T22:14:54,267 ERROR [RS_FLUSH_OPERATIONS-regionserver/bd6592c37bb2:0-1 {event_type=RS_FLUSH_REGIONS, pid=65}] handler.RSProcedureHandler(58): pid=65 java.io.IOException: Unable to complete flush {ENCODED => cdb3383a4019da5f5ef0568dfacd542e, NAME => 'TestAcidGuarantees,,1732486488674.cdb3383a4019da5f5ef0568dfacd542e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T22:14:54,268 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bd6592c37bb2:0-1 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=65 java.io.IOException: Unable to complete flush {ENCODED => cdb3383a4019da5f5ef0568dfacd542e, NAME => 'TestAcidGuarantees,,1732486488674.cdb3383a4019da5f5ef0568dfacd542e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T22:14:54,268 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37701 {}] master.HMaster(4177): Remote procedure failed, pid=65 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => cdb3383a4019da5f5ef0568dfacd542e, NAME => 'TestAcidGuarantees,,1732486488674.cdb3383a4019da5f5ef0568dfacd542e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => cdb3383a4019da5f5ef0568dfacd542e, NAME => 'TestAcidGuarantees,,1732486488674.cdb3383a4019da5f5ef0568dfacd542e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T22:14:54,281 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37701 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=64 2024-11-24T22:14:54,323 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40087 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cdb3383a4019da5f5ef0568dfacd542e, server=bd6592c37bb2,40087,1732486423713 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-24T22:14:54,324 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40087 {}] ipc.CallRunner(138): callId: 94 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40834 deadline: 1732486554321, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cdb3383a4019da5f5ef0568dfacd542e, server=bd6592c37bb2,40087,1732486423713 2024-11-24T22:14:54,324 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732486488674.cdb3383a4019da5f5ef0568dfacd542e., hostname=bd6592c37bb2,40087,1732486423713, seqNum=2 , the old value is region=TestAcidGuarantees,,1732486488674.cdb3383a4019da5f5ef0568dfacd542e., hostname=bd6592c37bb2,40087,1732486423713, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cdb3383a4019da5f5ef0568dfacd542e, server=bd6592c37bb2,40087,1732486423713 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T22:14:54,324 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732486488674.cdb3383a4019da5f5ef0568dfacd542e., hostname=bd6592c37bb2,40087,1732486423713, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cdb3383a4019da5f5ef0568dfacd542e, server=bd6592c37bb2,40087,1732486423713 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T22:14:54,324 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732486488674.cdb3383a4019da5f5ef0568dfacd542e., hostname=bd6592c37bb2,40087,1732486423713, seqNum=2 because the exception is null or not the one we care about 2024-11-24T22:14:54,420 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=40087 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=65 2024-11-24T22:14:54,420 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bd6592c37bb2:0-2 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732486488674.cdb3383a4019da5f5ef0568dfacd542e. 2024-11-24T22:14:54,421 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bd6592c37bb2:0-2 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732486488674.cdb3383a4019da5f5ef0568dfacd542e. as already flushing 2024-11-24T22:14:54,421 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bd6592c37bb2:0-2 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732486488674.cdb3383a4019da5f5ef0568dfacd542e. 2024-11-24T22:14:54,421 ERROR [RS_FLUSH_OPERATIONS-regionserver/bd6592c37bb2:0-2 {event_type=RS_FLUSH_REGIONS, pid=65}] handler.RSProcedureHandler(58): pid=65 java.io.IOException: Unable to complete flush {ENCODED => cdb3383a4019da5f5ef0568dfacd542e, NAME => 'TestAcidGuarantees,,1732486488674.cdb3383a4019da5f5ef0568dfacd542e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T22:14:54,421 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bd6592c37bb2:0-2 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=65 java.io.IOException: Unable to complete flush {ENCODED => cdb3383a4019da5f5ef0568dfacd542e, NAME => 'TestAcidGuarantees,,1732486488674.cdb3383a4019da5f5ef0568dfacd542e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T22:14:54,422 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37701 {}] master.HMaster(4177): Remote procedure failed, pid=65 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => cdb3383a4019da5f5ef0568dfacd542e, NAME => 'TestAcidGuarantees,,1732486488674.cdb3383a4019da5f5ef0568dfacd542e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => cdb3383a4019da5f5ef0568dfacd542e, NAME => 'TestAcidGuarantees,,1732486488674.cdb3383a4019da5f5ef0568dfacd542e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T22:14:54,479 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=31.31 KB at sequenceid=132 (bloomFilter=true), to=hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/cdb3383a4019da5f5ef0568dfacd542e/.tmp/C/112218e9c5d848ba91829de0208e34b2 2024-11-24T22:14:54,483 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/cdb3383a4019da5f5ef0568dfacd542e/.tmp/A/fa66f306f0f04c7d8adb9bd8978e52a5 as hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/cdb3383a4019da5f5ef0568dfacd542e/A/fa66f306f0f04c7d8adb9bd8978e52a5 2024-11-24T22:14:54,487 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/cdb3383a4019da5f5ef0568dfacd542e/A/fa66f306f0f04c7d8adb9bd8978e52a5, entries=200, sequenceid=132, filesize=14.1 K 2024-11-24T22:14:54,488 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/cdb3383a4019da5f5ef0568dfacd542e/.tmp/B/c6d4b3ff05d942349fdf2d2a9a078c6c as hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/cdb3383a4019da5f5ef0568dfacd542e/B/c6d4b3ff05d942349fdf2d2a9a078c6c 2024-11-24T22:14:54,493 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/cdb3383a4019da5f5ef0568dfacd542e/B/c6d4b3ff05d942349fdf2d2a9a078c6c, entries=150, sequenceid=132, filesize=11.8 K 2024-11-24T22:14:54,493 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/cdb3383a4019da5f5ef0568dfacd542e/.tmp/C/112218e9c5d848ba91829de0208e34b2 as hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/cdb3383a4019da5f5ef0568dfacd542e/C/112218e9c5d848ba91829de0208e34b2 2024-11-24T22:14:54,497 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/cdb3383a4019da5f5ef0568dfacd542e/C/112218e9c5d848ba91829de0208e34b2, entries=150, sequenceid=132, filesize=11.8 K 2024-11-24T22:14:54,498 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~93.93 KB/96180, heapSize ~279.61 KB/286320, currentSize=80.51 KB/82440 for cdb3383a4019da5f5ef0568dfacd542e in 875ms, sequenceid=132, compaction requested=true 2024-11-24T22:14:54,498 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for cdb3383a4019da5f5ef0568dfacd542e: 2024-11-24T22:14:54,498 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store cdb3383a4019da5f5ef0568dfacd542e:A, priority=-2147483648, current under compaction store size is 1 2024-11-24T22:14:54,498 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-24T22:14:54,498 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store cdb3383a4019da5f5ef0568dfacd542e:B, priority=-2147483648, current under compaction store size is 2 2024-11-24T22:14:54,498 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-24T22:14:54,498 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store cdb3383a4019da5f5ef0568dfacd542e:C, priority=-2147483648, current under compaction store size is 3 2024-11-24T22:14:54,498 DEBUG [RS:0;bd6592c37bb2:40087-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-24T22:14:54,498 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-24T22:14:54,498 DEBUG [RS:0;bd6592c37bb2:40087-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-24T22:14:54,500 DEBUG [RS:0;bd6592c37bb2:40087-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 38649 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-24T22:14:54,500 DEBUG [RS:0;bd6592c37bb2:40087-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36259 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-24T22:14:54,500 DEBUG [RS:0;bd6592c37bb2:40087-shortCompactions-0 {}] regionserver.HStore(1541): cdb3383a4019da5f5ef0568dfacd542e/A is initiating minor compaction (all files) 2024-11-24T22:14:54,500 DEBUG [RS:0;bd6592c37bb2:40087-longCompactions-0 {}] regionserver.HStore(1541): cdb3383a4019da5f5ef0568dfacd542e/B is initiating minor compaction (all files) 2024-11-24T22:14:54,500 INFO [RS:0;bd6592c37bb2:40087-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of cdb3383a4019da5f5ef0568dfacd542e/B in TestAcidGuarantees,,1732486488674.cdb3383a4019da5f5ef0568dfacd542e. 2024-11-24T22:14:54,500 INFO [RS:0;bd6592c37bb2:40087-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of cdb3383a4019da5f5ef0568dfacd542e/A in TestAcidGuarantees,,1732486488674.cdb3383a4019da5f5ef0568dfacd542e. 2024-11-24T22:14:54,500 INFO [RS:0;bd6592c37bb2:40087-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/cdb3383a4019da5f5ef0568dfacd542e/B/b68df242e91f4b989faa62390bff6f0d, hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/cdb3383a4019da5f5ef0568dfacd542e/B/65d304270e9042d58934bac821362e54, hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/cdb3383a4019da5f5ef0568dfacd542e/B/c6d4b3ff05d942349fdf2d2a9a078c6c] into tmpdir=hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/cdb3383a4019da5f5ef0568dfacd542e/.tmp, totalSize=35.4 K 2024-11-24T22:14:54,500 INFO [RS:0;bd6592c37bb2:40087-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/cdb3383a4019da5f5ef0568dfacd542e/A/97b091d10a0d4c3e9b134e1fc633706f, hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/cdb3383a4019da5f5ef0568dfacd542e/A/20eee9fcfcec4fceab01ad537875e97e, hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/cdb3383a4019da5f5ef0568dfacd542e/A/fa66f306f0f04c7d8adb9bd8978e52a5] into tmpdir=hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/cdb3383a4019da5f5ef0568dfacd542e/.tmp, totalSize=37.7 K 2024-11-24T22:14:54,500 DEBUG [RS:0;bd6592c37bb2:40087-longCompactions-0 {}] compactions.Compactor(225): Compacting b68df242e91f4b989faa62390bff6f0d, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=96, earliestPutTs=1732486492664 2024-11-24T22:14:54,501 DEBUG [RS:0;bd6592c37bb2:40087-shortCompactions-0 {}] compactions.Compactor(225): Compacting 97b091d10a0d4c3e9b134e1fc633706f, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=96, earliestPutTs=1732486492664 2024-11-24T22:14:54,501 DEBUG [RS:0;bd6592c37bb2:40087-longCompactions-0 {}] compactions.Compactor(225): Compacting 65d304270e9042d58934bac821362e54, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=112, earliestPutTs=1732486492857 2024-11-24T22:14:54,501 DEBUG [RS:0;bd6592c37bb2:40087-shortCompactions-0 {}] compactions.Compactor(225): Compacting 20eee9fcfcec4fceab01ad537875e97e, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=112, earliestPutTs=1732486492857 2024-11-24T22:14:54,501 DEBUG [RS:0;bd6592c37bb2:40087-longCompactions-0 {}] compactions.Compactor(225): Compacting c6d4b3ff05d942349fdf2d2a9a078c6c, keycount=150, bloomtype=ROW, size=11.8 K, encoding=NONE, compression=NONE, seqNum=132, earliestPutTs=1732486492946 2024-11-24T22:14:54,501 DEBUG [RS:0;bd6592c37bb2:40087-shortCompactions-0 {}] compactions.Compactor(225): Compacting fa66f306f0f04c7d8adb9bd8978e52a5, keycount=200, bloomtype=ROW, size=14.1 K, encoding=NONE, compression=NONE, seqNum=132, earliestPutTs=1732486492946 2024-11-24T22:14:54,512 INFO [RS:0;bd6592c37bb2:40087-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): cdb3383a4019da5f5ef0568dfacd542e#B#compaction#223 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-24T22:14:54,512 INFO [RS:0;bd6592c37bb2:40087-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): cdb3383a4019da5f5ef0568dfacd542e#A#compaction#222 average throughput is 2.18 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-24T22:14:54,512 DEBUG [RS:0;bd6592c37bb2:40087-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/cdb3383a4019da5f5ef0568dfacd542e/.tmp/B/a2122f90c78d4a8bb8b168164f69b0c9 is 50, key is test_row_0/B:col10/1732486493622/Put/seqid=0 2024-11-24T22:14:54,512 DEBUG [RS:0;bd6592c37bb2:40087-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/cdb3383a4019da5f5ef0568dfacd542e/.tmp/A/a05175f60c2046c8a2f1285b11374431 is 50, key is test_row_0/A:col10/1732486493622/Put/seqid=0 2024-11-24T22:14:54,522 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45929 is added to blk_1073742096_1272 (size=12359) 2024-11-24T22:14:54,528 DEBUG [RS:0;bd6592c37bb2:40087-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/cdb3383a4019da5f5ef0568dfacd542e/.tmp/B/a2122f90c78d4a8bb8b168164f69b0c9 as hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/cdb3383a4019da5f5ef0568dfacd542e/B/a2122f90c78d4a8bb8b168164f69b0c9 2024-11-24T22:14:54,530 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45929 is added to blk_1073742097_1273 (size=12359) 2024-11-24T22:14:54,533 INFO [RS:0;bd6592c37bb2:40087-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in cdb3383a4019da5f5ef0568dfacd542e/B of cdb3383a4019da5f5ef0568dfacd542e into a2122f90c78d4a8bb8b168164f69b0c9(size=12.1 K), total size for store is 12.1 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-24T22:14:54,534 DEBUG [RS:0;bd6592c37bb2:40087-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for cdb3383a4019da5f5ef0568dfacd542e: 2024-11-24T22:14:54,534 INFO [RS:0;bd6592c37bb2:40087-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732486488674.cdb3383a4019da5f5ef0568dfacd542e., storeName=cdb3383a4019da5f5ef0568dfacd542e/B, priority=13, startTime=1732486494498; duration=0sec 2024-11-24T22:14:54,534 DEBUG [RS:0;bd6592c37bb2:40087-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-24T22:14:54,534 DEBUG [RS:0;bd6592c37bb2:40087-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: cdb3383a4019da5f5ef0568dfacd542e:B 2024-11-24T22:14:54,534 DEBUG [RS:0;bd6592c37bb2:40087-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-24T22:14:54,535 DEBUG [RS:0;bd6592c37bb2:40087-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36259 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-24T22:14:54,535 DEBUG [RS:0;bd6592c37bb2:40087-longCompactions-0 {}] regionserver.HStore(1541): cdb3383a4019da5f5ef0568dfacd542e/C is initiating minor compaction (all files) 2024-11-24T22:14:54,535 INFO [RS:0;bd6592c37bb2:40087-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of cdb3383a4019da5f5ef0568dfacd542e/C in TestAcidGuarantees,,1732486488674.cdb3383a4019da5f5ef0568dfacd542e. 2024-11-24T22:14:54,535 INFO [RS:0;bd6592c37bb2:40087-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/cdb3383a4019da5f5ef0568dfacd542e/C/48dcc83ad5644812ba027d775c3a45f7, hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/cdb3383a4019da5f5ef0568dfacd542e/C/42c1b6cf23734fef8a3529416a6735e3, hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/cdb3383a4019da5f5ef0568dfacd542e/C/112218e9c5d848ba91829de0208e34b2] into tmpdir=hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/cdb3383a4019da5f5ef0568dfacd542e/.tmp, totalSize=35.4 K 2024-11-24T22:14:54,535 DEBUG [RS:0;bd6592c37bb2:40087-longCompactions-0 {}] compactions.Compactor(225): Compacting 48dcc83ad5644812ba027d775c3a45f7, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=96, earliestPutTs=1732486492664 2024-11-24T22:14:54,536 DEBUG [RS:0;bd6592c37bb2:40087-longCompactions-0 {}] compactions.Compactor(225): Compacting 42c1b6cf23734fef8a3529416a6735e3, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=112, earliestPutTs=1732486492857 2024-11-24T22:14:54,536 DEBUG [RS:0;bd6592c37bb2:40087-longCompactions-0 {}] compactions.Compactor(225): Compacting 112218e9c5d848ba91829de0208e34b2, keycount=150, bloomtype=ROW, size=11.8 K, encoding=NONE, compression=NONE, seqNum=132, earliestPutTs=1732486492946 2024-11-24T22:14:54,543 INFO [RS:0;bd6592c37bb2:40087-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): cdb3383a4019da5f5ef0568dfacd542e#C#compaction#224 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-24T22:14:54,544 DEBUG [RS:0;bd6592c37bb2:40087-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/cdb3383a4019da5f5ef0568dfacd542e/.tmp/C/b73271238a90407aae9295d57f98608a is 50, key is test_row_0/C:col10/1732486493622/Put/seqid=0 2024-11-24T22:14:54,548 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45929 is added to blk_1073742098_1274 (size=12359) 2024-11-24T22:14:54,555 DEBUG [RS:0;bd6592c37bb2:40087-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/cdb3383a4019da5f5ef0568dfacd542e/.tmp/C/b73271238a90407aae9295d57f98608a as hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/cdb3383a4019da5f5ef0568dfacd542e/C/b73271238a90407aae9295d57f98608a 2024-11-24T22:14:54,559 INFO [RS:0;bd6592c37bb2:40087-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in cdb3383a4019da5f5ef0568dfacd542e/C of cdb3383a4019da5f5ef0568dfacd542e into b73271238a90407aae9295d57f98608a(size=12.1 K), total size for store is 12.1 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-24T22:14:54,559 DEBUG [RS:0;bd6592c37bb2:40087-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for cdb3383a4019da5f5ef0568dfacd542e: 2024-11-24T22:14:54,559 INFO [RS:0;bd6592c37bb2:40087-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732486488674.cdb3383a4019da5f5ef0568dfacd542e., storeName=cdb3383a4019da5f5ef0568dfacd542e/C, priority=13, startTime=1732486494498; duration=0sec 2024-11-24T22:14:54,560 DEBUG [RS:0;bd6592c37bb2:40087-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-24T22:14:54,560 DEBUG [RS:0;bd6592c37bb2:40087-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: cdb3383a4019da5f5ef0568dfacd542e:C 2024-11-24T22:14:54,573 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=40087 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=65 2024-11-24T22:14:54,574 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bd6592c37bb2:0-0 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732486488674.cdb3383a4019da5f5ef0568dfacd542e. 2024-11-24T22:14:54,574 INFO [RS_FLUSH_OPERATIONS-regionserver/bd6592c37bb2:0-0 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.HRegion(2902): Flushing cdb3383a4019da5f5ef0568dfacd542e 3/3 column families, dataSize=80.51 KB heapSize=239.81 KB 2024-11-24T22:14:54,574 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bd6592c37bb2:0-0 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.CompactingMemStore(205): FLUSHING TO DISK cdb3383a4019da5f5ef0568dfacd542e, store=A 2024-11-24T22:14:54,574 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bd6592c37bb2:0-0 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-24T22:14:54,574 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bd6592c37bb2:0-0 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.CompactingMemStore(205): FLUSHING TO DISK cdb3383a4019da5f5ef0568dfacd542e, store=B 2024-11-24T22:14:54,574 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bd6592c37bb2:0-0 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-24T22:14:54,574 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bd6592c37bb2:0-0 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.CompactingMemStore(205): FLUSHING TO DISK cdb3383a4019da5f5ef0568dfacd542e, store=C 2024-11-24T22:14:54,575 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bd6592c37bb2:0-0 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-24T22:14:54,579 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bd6592c37bb2:0-0 {event_type=RS_FLUSH_REGIONS, pid=65}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/cdb3383a4019da5f5ef0568dfacd542e/.tmp/A/7ad153faedb44da490fdd7d12b6ba07b is 50, key is test_row_0/A:col10/1732486493679/Put/seqid=0 2024-11-24T22:14:54,584 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45929 is added to blk_1073742099_1275 (size=12151) 2024-11-24T22:14:54,790 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37701 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=64 2024-11-24T22:14:54,833 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40087 {}] regionserver.HRegion(8855): Flush requested on cdb3383a4019da5f5ef0568dfacd542e 2024-11-24T22:14:54,834 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732486488674.cdb3383a4019da5f5ef0568dfacd542e. as already flushing 2024-11-24T22:14:54,894 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40087 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cdb3383a4019da5f5ef0568dfacd542e, server=bd6592c37bb2,40087,1732486423713 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-24T22:14:54,894 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40087 {}] ipc.CallRunner(138): callId: 109 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40834 deadline: 1732486554889, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cdb3383a4019da5f5ef0568dfacd542e, server=bd6592c37bb2,40087,1732486423713 2024-11-24T22:14:54,895 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732486488674.cdb3383a4019da5f5ef0568dfacd542e., hostname=bd6592c37bb2,40087,1732486423713, seqNum=2 , the old value is region=TestAcidGuarantees,,1732486488674.cdb3383a4019da5f5ef0568dfacd542e., hostname=bd6592c37bb2,40087,1732486423713, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cdb3383a4019da5f5ef0568dfacd542e, server=bd6592c37bb2,40087,1732486423713 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T22:14:54,895 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732486488674.cdb3383a4019da5f5ef0568dfacd542e., hostname=bd6592c37bb2,40087,1732486423713, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cdb3383a4019da5f5ef0568dfacd542e, server=bd6592c37bb2,40087,1732486423713 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T22:14:54,895 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732486488674.cdb3383a4019da5f5ef0568dfacd542e., hostname=bd6592c37bb2,40087,1732486423713, seqNum=2 because the exception is null or not the one we care about 2024-11-24T22:14:54,937 DEBUG [RS:0;bd6592c37bb2:40087-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/cdb3383a4019da5f5ef0568dfacd542e/.tmp/A/a05175f60c2046c8a2f1285b11374431 as hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/cdb3383a4019da5f5ef0568dfacd542e/A/a05175f60c2046c8a2f1285b11374431 2024-11-24T22:14:54,941 INFO [RS:0;bd6592c37bb2:40087-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in cdb3383a4019da5f5ef0568dfacd542e/A of cdb3383a4019da5f5ef0568dfacd542e into a05175f60c2046c8a2f1285b11374431(size=12.1 K), total size for store is 12.1 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-24T22:14:54,941 DEBUG [RS:0;bd6592c37bb2:40087-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for cdb3383a4019da5f5ef0568dfacd542e: 2024-11-24T22:14:54,941 INFO [RS:0;bd6592c37bb2:40087-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732486488674.cdb3383a4019da5f5ef0568dfacd542e., storeName=cdb3383a4019da5f5ef0568dfacd542e/A, priority=13, startTime=1732486494498; duration=0sec 2024-11-24T22:14:54,941 DEBUG [RS:0;bd6592c37bb2:40087-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-24T22:14:54,941 DEBUG [RS:0;bd6592c37bb2:40087-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: cdb3383a4019da5f5ef0568dfacd542e:A 2024-11-24T22:14:54,984 INFO [RS_FLUSH_OPERATIONS-regionserver/bd6592c37bb2:0-0 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=26.84 KB at sequenceid=149 (bloomFilter=true), to=hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/cdb3383a4019da5f5ef0568dfacd542e/.tmp/A/7ad153faedb44da490fdd7d12b6ba07b 2024-11-24T22:14:54,992 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bd6592c37bb2:0-0 {event_type=RS_FLUSH_REGIONS, pid=65}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/cdb3383a4019da5f5ef0568dfacd542e/.tmp/B/2b02afeb3cba495ca56408dafcff1754 is 50, key is test_row_0/B:col10/1732486493679/Put/seqid=0 2024-11-24T22:14:55,005 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45929 is added to blk_1073742100_1276 (size=12151) 2024-11-24T22:14:55,005 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40087 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cdb3383a4019da5f5ef0568dfacd542e, server=bd6592c37bb2,40087,1732486423713 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-24T22:14:55,005 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40087 {}] ipc.CallRunner(138): callId: 110 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40834 deadline: 1732486555002, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cdb3383a4019da5f5ef0568dfacd542e, server=bd6592c37bb2,40087,1732486423713 2024-11-24T22:14:55,005 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732486488674.cdb3383a4019da5f5ef0568dfacd542e., hostname=bd6592c37bb2,40087,1732486423713, seqNum=2 , the old value is region=TestAcidGuarantees,,1732486488674.cdb3383a4019da5f5ef0568dfacd542e., hostname=bd6592c37bb2,40087,1732486423713, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cdb3383a4019da5f5ef0568dfacd542e, server=bd6592c37bb2,40087,1732486423713 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T22:14:55,006 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732486488674.cdb3383a4019da5f5ef0568dfacd542e., hostname=bd6592c37bb2,40087,1732486423713, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cdb3383a4019da5f5ef0568dfacd542e, server=bd6592c37bb2,40087,1732486423713 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T22:14:55,006 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732486488674.cdb3383a4019da5f5ef0568dfacd542e., hostname=bd6592c37bb2,40087,1732486423713, seqNum=2 because the exception is null or not the one we care about 2024-11-24T22:14:55,143 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40087 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cdb3383a4019da5f5ef0568dfacd542e, server=bd6592c37bb2,40087,1732486423713 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-24T22:14:55,143 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40087 {}] ipc.CallRunner(138): callId: 17 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40772 deadline: 1732486555140, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cdb3383a4019da5f5ef0568dfacd542e, server=bd6592c37bb2,40087,1732486423713 2024-11-24T22:14:55,144 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40087 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cdb3383a4019da5f5ef0568dfacd542e, server=bd6592c37bb2,40087,1732486423713 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-24T22:14:55,144 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40087 {}] ipc.CallRunner(138): callId: 16 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40842 deadline: 1732486555140, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cdb3383a4019da5f5ef0568dfacd542e, server=bd6592c37bb2,40087,1732486423713 2024-11-24T22:14:55,144 WARN [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRpcRetryingCaller(177): Call to bd6592c37bb2,40087,1732486423713 for 'test_row_2' in cdb3383a4019da5f5ef0568dfacd542e of TestAcidGuarantees failed, tries = 7, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 4188 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cdb3383a4019da5f5ef0568dfacd542e, server=bd6592c37bb2,40087,1732486423713 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor44.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cdb3383a4019da5f5ef0568dfacd542e, server=bd6592c37bb2,40087,1732486423713 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-24T22:14:55,144 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732486488674.cdb3383a4019da5f5ef0568dfacd542e., hostname=bd6592c37bb2,40087,1732486423713, seqNum=2 , the old value is region=TestAcidGuarantees,,1732486488674.cdb3383a4019da5f5ef0568dfacd542e., hostname=bd6592c37bb2,40087,1732486423713, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cdb3383a4019da5f5ef0568dfacd542e, server=bd6592c37bb2,40087,1732486423713 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T22:14:55,144 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732486488674.cdb3383a4019da5f5ef0568dfacd542e., hostname=bd6592c37bb2,40087,1732486423713, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cdb3383a4019da5f5ef0568dfacd542e, server=bd6592c37bb2,40087,1732486423713 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T22:14:55,144 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732486488674.cdb3383a4019da5f5ef0568dfacd542e., hostname=bd6592c37bb2,40087,1732486423713, seqNum=2 because the exception is null or not the one we care about 2024-11-24T22:14:55,144 WARN [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRpcRetryingCaller(177): Call to bd6592c37bb2,40087,1732486423713 for 'test_row_0' in cdb3383a4019da5f5ef0568dfacd542e of TestAcidGuarantees failed, tries = 7, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 4186 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cdb3383a4019da5f5ef0568dfacd542e, server=bd6592c37bb2,40087,1732486423713 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor44.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cdb3383a4019da5f5ef0568dfacd542e, server=bd6592c37bb2,40087,1732486423713 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-24T22:14:55,144 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732486488674.cdb3383a4019da5f5ef0568dfacd542e., hostname=bd6592c37bb2,40087,1732486423713, seqNum=2 , the old value is region=TestAcidGuarantees,,1732486488674.cdb3383a4019da5f5ef0568dfacd542e., hostname=bd6592c37bb2,40087,1732486423713, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cdb3383a4019da5f5ef0568dfacd542e, server=bd6592c37bb2,40087,1732486423713 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T22:14:55,145 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732486488674.cdb3383a4019da5f5ef0568dfacd542e., hostname=bd6592c37bb2,40087,1732486423713, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cdb3383a4019da5f5ef0568dfacd542e, server=bd6592c37bb2,40087,1732486423713 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T22:14:55,145 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732486488674.cdb3383a4019da5f5ef0568dfacd542e., hostname=bd6592c37bb2,40087,1732486423713, seqNum=2 because the exception is null or not the one we care about 2024-11-24T22:14:55,153 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40087 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cdb3383a4019da5f5ef0568dfacd542e, server=bd6592c37bb2,40087,1732486423713 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-24T22:14:55,154 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40087 {}] ipc.CallRunner(138): callId: 16 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40816 deadline: 1732486555151, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cdb3383a4019da5f5ef0568dfacd542e, server=bd6592c37bb2,40087,1732486423713 2024-11-24T22:14:55,154 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40087 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cdb3383a4019da5f5ef0568dfacd542e, server=bd6592c37bb2,40087,1732486423713 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-24T22:14:55,154 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40087 {}] ipc.CallRunner(138): callId: 16 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40806 deadline: 1732486555151, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cdb3383a4019da5f5ef0568dfacd542e, server=bd6592c37bb2,40087,1732486423713 2024-11-24T22:14:55,154 WARN [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRpcRetryingCaller(177): Call to bd6592c37bb2,40087,1732486423713 for 'test_row_2' in cdb3383a4019da5f5ef0568dfacd542e of TestAcidGuarantees failed, tries = 7, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 4196 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cdb3383a4019da5f5ef0568dfacd542e, server=bd6592c37bb2,40087,1732486423713 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor44.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cdb3383a4019da5f5ef0568dfacd542e, server=bd6592c37bb2,40087,1732486423713 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-24T22:14:55,154 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732486488674.cdb3383a4019da5f5ef0568dfacd542e., hostname=bd6592c37bb2,40087,1732486423713, seqNum=2 , the old value is region=TestAcidGuarantees,,1732486488674.cdb3383a4019da5f5ef0568dfacd542e., hostname=bd6592c37bb2,40087,1732486423713, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cdb3383a4019da5f5ef0568dfacd542e, server=bd6592c37bb2,40087,1732486423713 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T22:14:55,155 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732486488674.cdb3383a4019da5f5ef0568dfacd542e., hostname=bd6592c37bb2,40087,1732486423713, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cdb3383a4019da5f5ef0568dfacd542e, server=bd6592c37bb2,40087,1732486423713 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T22:14:55,155 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732486488674.cdb3383a4019da5f5ef0568dfacd542e., hostname=bd6592c37bb2,40087,1732486423713, seqNum=2 because the exception is null or not the one we care about 2024-11-24T22:14:55,154 WARN [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRpcRetryingCaller(177): Call to bd6592c37bb2,40087,1732486423713 for 'test_row_2' in cdb3383a4019da5f5ef0568dfacd542e of TestAcidGuarantees failed, tries = 7, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 4195 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cdb3383a4019da5f5ef0568dfacd542e, server=bd6592c37bb2,40087,1732486423713 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor44.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cdb3383a4019da5f5ef0568dfacd542e, server=bd6592c37bb2,40087,1732486423713 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-24T22:14:55,155 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732486488674.cdb3383a4019da5f5ef0568dfacd542e., hostname=bd6592c37bb2,40087,1732486423713, seqNum=2 , the old value is region=TestAcidGuarantees,,1732486488674.cdb3383a4019da5f5ef0568dfacd542e., hostname=bd6592c37bb2,40087,1732486423713, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cdb3383a4019da5f5ef0568dfacd542e, server=bd6592c37bb2,40087,1732486423713 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T22:14:55,155 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732486488674.cdb3383a4019da5f5ef0568dfacd542e., hostname=bd6592c37bb2,40087,1732486423713, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cdb3383a4019da5f5ef0568dfacd542e, server=bd6592c37bb2,40087,1732486423713 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T22:14:55,155 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732486488674.cdb3383a4019da5f5ef0568dfacd542e., hostname=bd6592c37bb2,40087,1732486423713, seqNum=2 because the exception is null or not the one we care about 2024-11-24T22:14:55,213 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40087 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cdb3383a4019da5f5ef0568dfacd542e, server=bd6592c37bb2,40087,1732486423713 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-24T22:14:55,214 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40087 {}] ipc.CallRunner(138): callId: 111 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40834 deadline: 1732486555210, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cdb3383a4019da5f5ef0568dfacd542e, server=bd6592c37bb2,40087,1732486423713 2024-11-24T22:14:55,214 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732486488674.cdb3383a4019da5f5ef0568dfacd542e., hostname=bd6592c37bb2,40087,1732486423713, seqNum=2 , the old value is region=TestAcidGuarantees,,1732486488674.cdb3383a4019da5f5ef0568dfacd542e., hostname=bd6592c37bb2,40087,1732486423713, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cdb3383a4019da5f5ef0568dfacd542e, server=bd6592c37bb2,40087,1732486423713 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T22:14:55,214 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732486488674.cdb3383a4019da5f5ef0568dfacd542e., hostname=bd6592c37bb2,40087,1732486423713, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cdb3383a4019da5f5ef0568dfacd542e, server=bd6592c37bb2,40087,1732486423713 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T22:14:55,214 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732486488674.cdb3383a4019da5f5ef0568dfacd542e., hostname=bd6592c37bb2,40087,1732486423713, seqNum=2 because the exception is null or not the one we care about 2024-11-24T22:14:55,405 INFO [RS_FLUSH_OPERATIONS-regionserver/bd6592c37bb2:0-0 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=26.84 KB at sequenceid=149 (bloomFilter=true), to=hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/cdb3383a4019da5f5ef0568dfacd542e/.tmp/B/2b02afeb3cba495ca56408dafcff1754 2024-11-24T22:14:55,415 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bd6592c37bb2:0-0 {event_type=RS_FLUSH_REGIONS, pid=65}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/cdb3383a4019da5f5ef0568dfacd542e/.tmp/C/062cecfaead74cc6864f360d41e8e99f is 50, key is test_row_0/C:col10/1732486493679/Put/seqid=0 2024-11-24T22:14:55,422 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45929 is added to blk_1073742101_1277 (size=12151) 2024-11-24T22:14:55,423 INFO [RS_FLUSH_OPERATIONS-regionserver/bd6592c37bb2:0-0 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=26.84 KB at sequenceid=149 (bloomFilter=true), to=hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/cdb3383a4019da5f5ef0568dfacd542e/.tmp/C/062cecfaead74cc6864f360d41e8e99f 2024-11-24T22:14:55,429 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bd6592c37bb2:0-0 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/cdb3383a4019da5f5ef0568dfacd542e/.tmp/A/7ad153faedb44da490fdd7d12b6ba07b as hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/cdb3383a4019da5f5ef0568dfacd542e/A/7ad153faedb44da490fdd7d12b6ba07b 2024-11-24T22:14:55,433 INFO [RS_FLUSH_OPERATIONS-regionserver/bd6592c37bb2:0-0 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/cdb3383a4019da5f5ef0568dfacd542e/A/7ad153faedb44da490fdd7d12b6ba07b, entries=150, sequenceid=149, filesize=11.9 K 2024-11-24T22:14:55,434 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bd6592c37bb2:0-0 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/cdb3383a4019da5f5ef0568dfacd542e/.tmp/B/2b02afeb3cba495ca56408dafcff1754 as hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/cdb3383a4019da5f5ef0568dfacd542e/B/2b02afeb3cba495ca56408dafcff1754 2024-11-24T22:14:55,437 INFO [RS_FLUSH_OPERATIONS-regionserver/bd6592c37bb2:0-0 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/cdb3383a4019da5f5ef0568dfacd542e/B/2b02afeb3cba495ca56408dafcff1754, entries=150, sequenceid=149, filesize=11.9 K 2024-11-24T22:14:55,438 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bd6592c37bb2:0-0 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/cdb3383a4019da5f5ef0568dfacd542e/.tmp/C/062cecfaead74cc6864f360d41e8e99f as hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/cdb3383a4019da5f5ef0568dfacd542e/C/062cecfaead74cc6864f360d41e8e99f 2024-11-24T22:14:55,442 INFO [RS_FLUSH_OPERATIONS-regionserver/bd6592c37bb2:0-0 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/cdb3383a4019da5f5ef0568dfacd542e/C/062cecfaead74cc6864f360d41e8e99f, entries=150, sequenceid=149, filesize=11.9 K 2024-11-24T22:14:55,442 INFO [RS_FLUSH_OPERATIONS-regionserver/bd6592c37bb2:0-0 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.HRegion(3140): Finished flush of dataSize ~80.51 KB/82440, heapSize ~239.77 KB/245520, currentSize=93.93 KB/96180 for cdb3383a4019da5f5ef0568dfacd542e in 868ms, sequenceid=149, compaction requested=false 2024-11-24T22:14:55,442 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bd6592c37bb2:0-0 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.HRegion(2603): Flush status journal for cdb3383a4019da5f5ef0568dfacd542e: 2024-11-24T22:14:55,442 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bd6592c37bb2:0-0 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732486488674.cdb3383a4019da5f5ef0568dfacd542e. 2024-11-24T22:14:55,442 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bd6592c37bb2:0-0 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=65 2024-11-24T22:14:55,443 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37701 {}] master.HMaster(4169): Remote procedure done, pid=65 2024-11-24T22:14:55,446 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=65, resume processing ppid=64 2024-11-24T22:14:55,446 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=65, ppid=64, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.7890 sec 2024-11-24T22:14:55,448 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=64, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=64, table=TestAcidGuarantees in 1.7940 sec 2024-11-24T22:14:55,524 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40087 {}] regionserver.HRegion(8855): Flush requested on cdb3383a4019da5f5ef0568dfacd542e 2024-11-24T22:14:55,524 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing cdb3383a4019da5f5ef0568dfacd542e 3/3 column families, dataSize=100.63 KB heapSize=299.58 KB 2024-11-24T22:14:55,525 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK cdb3383a4019da5f5ef0568dfacd542e, store=A 2024-11-24T22:14:55,525 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-24T22:14:55,525 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK cdb3383a4019da5f5ef0568dfacd542e, store=B 2024-11-24T22:14:55,525 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-24T22:14:55,525 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK cdb3383a4019da5f5ef0568dfacd542e, store=C 2024-11-24T22:14:55,525 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-24T22:14:55,530 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/cdb3383a4019da5f5ef0568dfacd542e/.tmp/A/34f1f60d5fbd41e9a65ea799007f6da7 is 50, key is test_row_0/A:col10/1732486495523/Put/seqid=0 2024-11-24T22:14:55,534 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45929 is added to blk_1073742102_1278 (size=14541) 2024-11-24T22:14:55,581 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40087 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cdb3383a4019da5f5ef0568dfacd542e, server=bd6592c37bb2,40087,1732486423713 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-24T22:14:55,581 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40087 {}] ipc.CallRunner(138): callId: 124 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40834 deadline: 1732486555576, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cdb3383a4019da5f5ef0568dfacd542e, server=bd6592c37bb2,40087,1732486423713 2024-11-24T22:14:55,581 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732486488674.cdb3383a4019da5f5ef0568dfacd542e., hostname=bd6592c37bb2,40087,1732486423713, seqNum=2 , the old value is region=TestAcidGuarantees,,1732486488674.cdb3383a4019da5f5ef0568dfacd542e., hostname=bd6592c37bb2,40087,1732486423713, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cdb3383a4019da5f5ef0568dfacd542e, server=bd6592c37bb2,40087,1732486423713 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T22:14:55,581 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732486488674.cdb3383a4019da5f5ef0568dfacd542e., hostname=bd6592c37bb2,40087,1732486423713, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cdb3383a4019da5f5ef0568dfacd542e, server=bd6592c37bb2,40087,1732486423713 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T22:14:55,581 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732486488674.cdb3383a4019da5f5ef0568dfacd542e., hostname=bd6592c37bb2,40087,1732486423713, seqNum=2 because the exception is null or not the one we care about 2024-11-24T22:14:55,693 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40087 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cdb3383a4019da5f5ef0568dfacd542e, server=bd6592c37bb2,40087,1732486423713 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-24T22:14:55,693 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40087 {}] ipc.CallRunner(138): callId: 125 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40834 deadline: 1732486555691, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cdb3383a4019da5f5ef0568dfacd542e, server=bd6592c37bb2,40087,1732486423713 2024-11-24T22:14:55,693 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732486488674.cdb3383a4019da5f5ef0568dfacd542e., hostname=bd6592c37bb2,40087,1732486423713, seqNum=2 , the old value is region=TestAcidGuarantees,,1732486488674.cdb3383a4019da5f5ef0568dfacd542e., hostname=bd6592c37bb2,40087,1732486423713, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cdb3383a4019da5f5ef0568dfacd542e, server=bd6592c37bb2,40087,1732486423713 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T22:14:55,693 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732486488674.cdb3383a4019da5f5ef0568dfacd542e., hostname=bd6592c37bb2,40087,1732486423713, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cdb3383a4019da5f5ef0568dfacd542e, server=bd6592c37bb2,40087,1732486423713 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T22:14:55,693 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732486488674.cdb3383a4019da5f5ef0568dfacd542e., hostname=bd6592c37bb2,40087,1732486423713, seqNum=2 because the exception is null or not the one we care about 2024-11-24T22:14:55,801 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37701 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=64 2024-11-24T22:14:55,801 INFO [RPCClient-NioEventLoopGroup-4-6 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-24T22:14:55,802 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37701 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-24T22:14:55,803 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37701 {}] procedure2.ProcedureExecutor(1139): Stored pid=66, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=66, table=TestAcidGuarantees 2024-11-24T22:14:55,804 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37701 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=66 2024-11-24T22:14:55,805 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=66, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=66, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-24T22:14:55,805 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=66, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=66, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-24T22:14:55,805 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=67, ppid=66, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-24T22:14:55,902 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40087 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cdb3383a4019da5f5ef0568dfacd542e, server=bd6592c37bb2,40087,1732486423713 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-24T22:14:55,902 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40087 {}] ipc.CallRunner(138): callId: 126 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40834 deadline: 1732486555901, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cdb3383a4019da5f5ef0568dfacd542e, server=bd6592c37bb2,40087,1732486423713 2024-11-24T22:14:55,903 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732486488674.cdb3383a4019da5f5ef0568dfacd542e., hostname=bd6592c37bb2,40087,1732486423713, seqNum=2 , the old value is region=TestAcidGuarantees,,1732486488674.cdb3383a4019da5f5ef0568dfacd542e., hostname=bd6592c37bb2,40087,1732486423713, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cdb3383a4019da5f5ef0568dfacd542e, server=bd6592c37bb2,40087,1732486423713 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T22:14:55,903 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732486488674.cdb3383a4019da5f5ef0568dfacd542e., hostname=bd6592c37bb2,40087,1732486423713, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cdb3383a4019da5f5ef0568dfacd542e, server=bd6592c37bb2,40087,1732486423713 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T22:14:55,903 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732486488674.cdb3383a4019da5f5ef0568dfacd542e., hostname=bd6592c37bb2,40087,1732486423713, seqNum=2 because the exception is null or not the one we care about 2024-11-24T22:14:55,911 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37701 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=66 2024-11-24T22:14:55,935 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=33.54 KB at sequenceid=168 (bloomFilter=true), to=hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/cdb3383a4019da5f5ef0568dfacd542e/.tmp/A/34f1f60d5fbd41e9a65ea799007f6da7 2024-11-24T22:14:55,942 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/cdb3383a4019da5f5ef0568dfacd542e/.tmp/B/99967a88476643f29f948c0d2ad668f0 is 50, key is test_row_0/B:col10/1732486495523/Put/seqid=0 2024-11-24T22:14:55,946 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45929 is added to blk_1073742103_1279 (size=12151) 2024-11-24T22:14:55,957 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=40087 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=67 2024-11-24T22:14:55,957 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bd6592c37bb2:0-1 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732486488674.cdb3383a4019da5f5ef0568dfacd542e. 2024-11-24T22:14:55,958 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bd6592c37bb2:0-1 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732486488674.cdb3383a4019da5f5ef0568dfacd542e. as already flushing 2024-11-24T22:14:55,958 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bd6592c37bb2:0-1 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732486488674.cdb3383a4019da5f5ef0568dfacd542e. 2024-11-24T22:14:55,958 ERROR [RS_FLUSH_OPERATIONS-regionserver/bd6592c37bb2:0-1 {event_type=RS_FLUSH_REGIONS, pid=67}] handler.RSProcedureHandler(58): pid=67 java.io.IOException: Unable to complete flush {ENCODED => cdb3383a4019da5f5ef0568dfacd542e, NAME => 'TestAcidGuarantees,,1732486488674.cdb3383a4019da5f5ef0568dfacd542e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T22:14:55,958 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bd6592c37bb2:0-1 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=67 java.io.IOException: Unable to complete flush {ENCODED => cdb3383a4019da5f5ef0568dfacd542e, NAME => 'TestAcidGuarantees,,1732486488674.cdb3383a4019da5f5ef0568dfacd542e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T22:14:55,959 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37701 {}] master.HMaster(4177): Remote procedure failed, pid=67 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => cdb3383a4019da5f5ef0568dfacd542e, NAME => 'TestAcidGuarantees,,1732486488674.cdb3383a4019da5f5ef0568dfacd542e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => cdb3383a4019da5f5ef0568dfacd542e, NAME => 'TestAcidGuarantees,,1732486488674.cdb3383a4019da5f5ef0568dfacd542e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T22:14:56,111 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=40087 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=67 2024-11-24T22:14:56,111 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bd6592c37bb2:0-2 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732486488674.cdb3383a4019da5f5ef0568dfacd542e. 2024-11-24T22:14:56,111 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bd6592c37bb2:0-2 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732486488674.cdb3383a4019da5f5ef0568dfacd542e. as already flushing 2024-11-24T22:14:56,111 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bd6592c37bb2:0-2 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732486488674.cdb3383a4019da5f5ef0568dfacd542e. 2024-11-24T22:14:56,111 ERROR [RS_FLUSH_OPERATIONS-regionserver/bd6592c37bb2:0-2 {event_type=RS_FLUSH_REGIONS, pid=67}] handler.RSProcedureHandler(58): pid=67 java.io.IOException: Unable to complete flush {ENCODED => cdb3383a4019da5f5ef0568dfacd542e, NAME => 'TestAcidGuarantees,,1732486488674.cdb3383a4019da5f5ef0568dfacd542e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T22:14:56,111 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bd6592c37bb2:0-2 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=67 java.io.IOException: Unable to complete flush {ENCODED => cdb3383a4019da5f5ef0568dfacd542e, NAME => 'TestAcidGuarantees,,1732486488674.cdb3383a4019da5f5ef0568dfacd542e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T22:14:56,112 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37701 {}] master.HMaster(4177): Remote procedure failed, pid=67 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => cdb3383a4019da5f5ef0568dfacd542e, NAME => 'TestAcidGuarantees,,1732486488674.cdb3383a4019da5f5ef0568dfacd542e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => cdb3383a4019da5f5ef0568dfacd542e, NAME => 'TestAcidGuarantees,,1732486488674.cdb3383a4019da5f5ef0568dfacd542e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T22:14:56,121 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37701 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=66 2024-11-24T22:14:56,213 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40087 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cdb3383a4019da5f5ef0568dfacd542e, server=bd6592c37bb2,40087,1732486423713 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-24T22:14:56,213 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40087 {}] ipc.CallRunner(138): callId: 127 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40834 deadline: 1732486556211, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cdb3383a4019da5f5ef0568dfacd542e, server=bd6592c37bb2,40087,1732486423713 2024-11-24T22:14:56,213 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732486488674.cdb3383a4019da5f5ef0568dfacd542e., hostname=bd6592c37bb2,40087,1732486423713, seqNum=2 , the old value is region=TestAcidGuarantees,,1732486488674.cdb3383a4019da5f5ef0568dfacd542e., hostname=bd6592c37bb2,40087,1732486423713, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cdb3383a4019da5f5ef0568dfacd542e, server=bd6592c37bb2,40087,1732486423713 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T22:14:56,213 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732486488674.cdb3383a4019da5f5ef0568dfacd542e., hostname=bd6592c37bb2,40087,1732486423713, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cdb3383a4019da5f5ef0568dfacd542e, server=bd6592c37bb2,40087,1732486423713 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T22:14:56,213 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732486488674.cdb3383a4019da5f5ef0568dfacd542e., hostname=bd6592c37bb2,40087,1732486423713, seqNum=2 because the exception is null or not the one we care about 2024-11-24T22:14:56,264 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=40087 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=67 2024-11-24T22:14:56,264 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bd6592c37bb2:0-0 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732486488674.cdb3383a4019da5f5ef0568dfacd542e. 2024-11-24T22:14:56,264 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bd6592c37bb2:0-0 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732486488674.cdb3383a4019da5f5ef0568dfacd542e. as already flushing 2024-11-24T22:14:56,264 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bd6592c37bb2:0-0 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732486488674.cdb3383a4019da5f5ef0568dfacd542e. 2024-11-24T22:14:56,264 ERROR [RS_FLUSH_OPERATIONS-regionserver/bd6592c37bb2:0-0 {event_type=RS_FLUSH_REGIONS, pid=67}] handler.RSProcedureHandler(58): pid=67 java.io.IOException: Unable to complete flush {ENCODED => cdb3383a4019da5f5ef0568dfacd542e, NAME => 'TestAcidGuarantees,,1732486488674.cdb3383a4019da5f5ef0568dfacd542e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T22:14:56,264 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bd6592c37bb2:0-0 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=67 java.io.IOException: Unable to complete flush {ENCODED => cdb3383a4019da5f5ef0568dfacd542e, NAME => 'TestAcidGuarantees,,1732486488674.cdb3383a4019da5f5ef0568dfacd542e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T22:14:56,265 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37701 {}] master.HMaster(4177): Remote procedure failed, pid=67 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => cdb3383a4019da5f5ef0568dfacd542e, NAME => 'TestAcidGuarantees,,1732486488674.cdb3383a4019da5f5ef0568dfacd542e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => cdb3383a4019da5f5ef0568dfacd542e, NAME => 'TestAcidGuarantees,,1732486488674.cdb3383a4019da5f5ef0568dfacd542e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T22:14:56,347 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=33.54 KB at sequenceid=168 (bloomFilter=true), to=hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/cdb3383a4019da5f5ef0568dfacd542e/.tmp/B/99967a88476643f29f948c0d2ad668f0 2024-11-24T22:14:56,354 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/cdb3383a4019da5f5ef0568dfacd542e/.tmp/C/6408a0d1112d4fa9853e63edbdd01c09 is 50, key is test_row_0/C:col10/1732486495523/Put/seqid=0 2024-11-24T22:14:56,357 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45929 is added to blk_1073742104_1280 (size=12151) 2024-11-24T22:14:56,417 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=40087 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=67 2024-11-24T22:14:56,417 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bd6592c37bb2:0-1 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732486488674.cdb3383a4019da5f5ef0568dfacd542e. 2024-11-24T22:14:56,417 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bd6592c37bb2:0-1 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732486488674.cdb3383a4019da5f5ef0568dfacd542e. as already flushing 2024-11-24T22:14:56,417 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bd6592c37bb2:0-1 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732486488674.cdb3383a4019da5f5ef0568dfacd542e. 2024-11-24T22:14:56,417 ERROR [RS_FLUSH_OPERATIONS-regionserver/bd6592c37bb2:0-1 {event_type=RS_FLUSH_REGIONS, pid=67}] handler.RSProcedureHandler(58): pid=67 java.io.IOException: Unable to complete flush {ENCODED => cdb3383a4019da5f5ef0568dfacd542e, NAME => 'TestAcidGuarantees,,1732486488674.cdb3383a4019da5f5ef0568dfacd542e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T22:14:56,417 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bd6592c37bb2:0-1 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=67 java.io.IOException: Unable to complete flush {ENCODED => cdb3383a4019da5f5ef0568dfacd542e, NAME => 'TestAcidGuarantees,,1732486488674.cdb3383a4019da5f5ef0568dfacd542e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T22:14:56,418 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37701 {}] master.HMaster(4177): Remote procedure failed, pid=67 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => cdb3383a4019da5f5ef0568dfacd542e, NAME => 'TestAcidGuarantees,,1732486488674.cdb3383a4019da5f5ef0568dfacd542e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => cdb3383a4019da5f5ef0568dfacd542e, NAME => 'TestAcidGuarantees,,1732486488674.cdb3383a4019da5f5ef0568dfacd542e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T22:14:56,430 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37701 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=66 2024-11-24T22:14:56,570 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=40087 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=67 2024-11-24T22:14:56,570 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bd6592c37bb2:0-2 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732486488674.cdb3383a4019da5f5ef0568dfacd542e. 2024-11-24T22:14:56,570 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bd6592c37bb2:0-2 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732486488674.cdb3383a4019da5f5ef0568dfacd542e. as already flushing 2024-11-24T22:14:56,570 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bd6592c37bb2:0-2 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732486488674.cdb3383a4019da5f5ef0568dfacd542e. 2024-11-24T22:14:56,570 ERROR [RS_FLUSH_OPERATIONS-regionserver/bd6592c37bb2:0-2 {event_type=RS_FLUSH_REGIONS, pid=67}] handler.RSProcedureHandler(58): pid=67 java.io.IOException: Unable to complete flush {ENCODED => cdb3383a4019da5f5ef0568dfacd542e, NAME => 'TestAcidGuarantees,,1732486488674.cdb3383a4019da5f5ef0568dfacd542e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T22:14:56,570 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bd6592c37bb2:0-2 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=67 java.io.IOException: Unable to complete flush {ENCODED => cdb3383a4019da5f5ef0568dfacd542e, NAME => 'TestAcidGuarantees,,1732486488674.cdb3383a4019da5f5ef0568dfacd542e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T22:14:56,571 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37701 {}] master.HMaster(4177): Remote procedure failed, pid=67 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => cdb3383a4019da5f5ef0568dfacd542e, NAME => 'TestAcidGuarantees,,1732486488674.cdb3383a4019da5f5ef0568dfacd542e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => cdb3383a4019da5f5ef0568dfacd542e, NAME => 'TestAcidGuarantees,,1732486488674.cdb3383a4019da5f5ef0568dfacd542e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T22:14:56,723 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=40087 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=67 2024-11-24T22:14:56,723 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bd6592c37bb2:0-0 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732486488674.cdb3383a4019da5f5ef0568dfacd542e. 2024-11-24T22:14:56,723 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bd6592c37bb2:0-0 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732486488674.cdb3383a4019da5f5ef0568dfacd542e. as already flushing 2024-11-24T22:14:56,723 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bd6592c37bb2:0-0 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732486488674.cdb3383a4019da5f5ef0568dfacd542e. 2024-11-24T22:14:56,723 ERROR [RS_FLUSH_OPERATIONS-regionserver/bd6592c37bb2:0-0 {event_type=RS_FLUSH_REGIONS, pid=67}] handler.RSProcedureHandler(58): pid=67 java.io.IOException: Unable to complete flush {ENCODED => cdb3383a4019da5f5ef0568dfacd542e, NAME => 'TestAcidGuarantees,,1732486488674.cdb3383a4019da5f5ef0568dfacd542e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T22:14:56,724 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bd6592c37bb2:0-0 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=67 java.io.IOException: Unable to complete flush {ENCODED => cdb3383a4019da5f5ef0568dfacd542e, NAME => 'TestAcidGuarantees,,1732486488674.cdb3383a4019da5f5ef0568dfacd542e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T22:14:56,724 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40087 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cdb3383a4019da5f5ef0568dfacd542e, server=bd6592c37bb2,40087,1732486423713 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-24T22:14:56,724 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40087 {}] ipc.CallRunner(138): callId: 128 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40834 deadline: 1732486556721, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cdb3383a4019da5f5ef0568dfacd542e, server=bd6592c37bb2,40087,1732486423713 2024-11-24T22:14:56,725 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732486488674.cdb3383a4019da5f5ef0568dfacd542e., hostname=bd6592c37bb2,40087,1732486423713, seqNum=2 , the old value is region=TestAcidGuarantees,,1732486488674.cdb3383a4019da5f5ef0568dfacd542e., hostname=bd6592c37bb2,40087,1732486423713, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cdb3383a4019da5f5ef0568dfacd542e, server=bd6592c37bb2,40087,1732486423713 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T22:14:56,725 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732486488674.cdb3383a4019da5f5ef0568dfacd542e., hostname=bd6592c37bb2,40087,1732486423713, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cdb3383a4019da5f5ef0568dfacd542e, server=bd6592c37bb2,40087,1732486423713 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T22:14:56,725 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732486488674.cdb3383a4019da5f5ef0568dfacd542e., hostname=bd6592c37bb2,40087,1732486423713, seqNum=2 because the exception is null or not the one we care about 2024-11-24T22:14:56,725 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37701 {}] master.HMaster(4177): Remote procedure failed, pid=67 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => cdb3383a4019da5f5ef0568dfacd542e, NAME => 'TestAcidGuarantees,,1732486488674.cdb3383a4019da5f5ef0568dfacd542e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => cdb3383a4019da5f5ef0568dfacd542e, NAME => 'TestAcidGuarantees,,1732486488674.cdb3383a4019da5f5ef0568dfacd542e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T22:14:56,758 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=33.54 KB at sequenceid=168 (bloomFilter=true), to=hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/cdb3383a4019da5f5ef0568dfacd542e/.tmp/C/6408a0d1112d4fa9853e63edbdd01c09 2024-11-24T22:14:56,762 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/cdb3383a4019da5f5ef0568dfacd542e/.tmp/A/34f1f60d5fbd41e9a65ea799007f6da7 as hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/cdb3383a4019da5f5ef0568dfacd542e/A/34f1f60d5fbd41e9a65ea799007f6da7 2024-11-24T22:14:56,766 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/cdb3383a4019da5f5ef0568dfacd542e/A/34f1f60d5fbd41e9a65ea799007f6da7, entries=200, sequenceid=168, filesize=14.2 K 2024-11-24T22:14:56,767 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/cdb3383a4019da5f5ef0568dfacd542e/.tmp/B/99967a88476643f29f948c0d2ad668f0 as hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/cdb3383a4019da5f5ef0568dfacd542e/B/99967a88476643f29f948c0d2ad668f0 2024-11-24T22:14:56,771 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/cdb3383a4019da5f5ef0568dfacd542e/B/99967a88476643f29f948c0d2ad668f0, entries=150, sequenceid=168, filesize=11.9 K 2024-11-24T22:14:56,772 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/cdb3383a4019da5f5ef0568dfacd542e/.tmp/C/6408a0d1112d4fa9853e63edbdd01c09 as hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/cdb3383a4019da5f5ef0568dfacd542e/C/6408a0d1112d4fa9853e63edbdd01c09 2024-11-24T22:14:56,776 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/cdb3383a4019da5f5ef0568dfacd542e/C/6408a0d1112d4fa9853e63edbdd01c09, entries=150, sequenceid=168, filesize=11.9 K 2024-11-24T22:14:56,776 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~100.63 KB/103050, heapSize ~299.53 KB/306720, currentSize=73.80 KB/75570 for cdb3383a4019da5f5ef0568dfacd542e in 1252ms, sequenceid=168, compaction requested=true 2024-11-24T22:14:56,777 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for cdb3383a4019da5f5ef0568dfacd542e: 2024-11-24T22:14:56,777 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store cdb3383a4019da5f5ef0568dfacd542e:A, priority=-2147483648, current under compaction store size is 1 2024-11-24T22:14:56,777 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-24T22:14:56,777 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store cdb3383a4019da5f5ef0568dfacd542e:B, priority=-2147483648, current under compaction store size is 2 2024-11-24T22:14:56,777 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-24T22:14:56,777 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store cdb3383a4019da5f5ef0568dfacd542e:C, priority=-2147483648, current under compaction store size is 3 2024-11-24T22:14:56,777 DEBUG [RS:0;bd6592c37bb2:40087-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-24T22:14:56,777 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-24T22:14:56,777 DEBUG [RS:0;bd6592c37bb2:40087-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-24T22:14:56,778 DEBUG [RS:0;bd6592c37bb2:40087-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 39051 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-24T22:14:56,778 DEBUG [RS:0;bd6592c37bb2:40087-shortCompactions-0 {}] regionserver.HStore(1541): cdb3383a4019da5f5ef0568dfacd542e/A is initiating minor compaction (all files) 2024-11-24T22:14:56,778 INFO [RS:0;bd6592c37bb2:40087-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of cdb3383a4019da5f5ef0568dfacd542e/A in TestAcidGuarantees,,1732486488674.cdb3383a4019da5f5ef0568dfacd542e. 2024-11-24T22:14:56,778 INFO [RS:0;bd6592c37bb2:40087-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/cdb3383a4019da5f5ef0568dfacd542e/A/a05175f60c2046c8a2f1285b11374431, hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/cdb3383a4019da5f5ef0568dfacd542e/A/7ad153faedb44da490fdd7d12b6ba07b, hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/cdb3383a4019da5f5ef0568dfacd542e/A/34f1f60d5fbd41e9a65ea799007f6da7] into tmpdir=hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/cdb3383a4019da5f5ef0568dfacd542e/.tmp, totalSize=38.1 K 2024-11-24T22:14:56,778 DEBUG [RS:0;bd6592c37bb2:40087-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36661 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-24T22:14:56,778 DEBUG [RS:0;bd6592c37bb2:40087-longCompactions-0 {}] regionserver.HStore(1541): cdb3383a4019da5f5ef0568dfacd542e/B is initiating minor compaction (all files) 2024-11-24T22:14:56,778 DEBUG [RS:0;bd6592c37bb2:40087-shortCompactions-0 {}] compactions.Compactor(225): Compacting a05175f60c2046c8a2f1285b11374431, keycount=150, bloomtype=ROW, size=12.1 K, encoding=NONE, compression=NONE, seqNum=132, earliestPutTs=1732486492946 2024-11-24T22:14:56,778 INFO [RS:0;bd6592c37bb2:40087-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of cdb3383a4019da5f5ef0568dfacd542e/B in TestAcidGuarantees,,1732486488674.cdb3383a4019da5f5ef0568dfacd542e. 2024-11-24T22:14:56,779 INFO [RS:0;bd6592c37bb2:40087-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/cdb3383a4019da5f5ef0568dfacd542e/B/a2122f90c78d4a8bb8b168164f69b0c9, hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/cdb3383a4019da5f5ef0568dfacd542e/B/2b02afeb3cba495ca56408dafcff1754, hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/cdb3383a4019da5f5ef0568dfacd542e/B/99967a88476643f29f948c0d2ad668f0] into tmpdir=hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/cdb3383a4019da5f5ef0568dfacd542e/.tmp, totalSize=35.8 K 2024-11-24T22:14:56,779 DEBUG [RS:0;bd6592c37bb2:40087-shortCompactions-0 {}] compactions.Compactor(225): Compacting 7ad153faedb44da490fdd7d12b6ba07b, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=149, earliestPutTs=1732486493668 2024-11-24T22:14:56,779 DEBUG [RS:0;bd6592c37bb2:40087-longCompactions-0 {}] compactions.Compactor(225): Compacting a2122f90c78d4a8bb8b168164f69b0c9, keycount=150, bloomtype=ROW, size=12.1 K, encoding=NONE, compression=NONE, seqNum=132, earliestPutTs=1732486492946 2024-11-24T22:14:56,780 DEBUG [RS:0;bd6592c37bb2:40087-shortCompactions-0 {}] compactions.Compactor(225): Compacting 34f1f60d5fbd41e9a65ea799007f6da7, keycount=200, bloomtype=ROW, size=14.2 K, encoding=NONE, compression=NONE, seqNum=168, earliestPutTs=1732486494877 2024-11-24T22:14:56,780 DEBUG [RS:0;bd6592c37bb2:40087-longCompactions-0 {}] compactions.Compactor(225): Compacting 2b02afeb3cba495ca56408dafcff1754, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=149, earliestPutTs=1732486493668 2024-11-24T22:14:56,780 DEBUG [RS:0;bd6592c37bb2:40087-longCompactions-0 {}] compactions.Compactor(225): Compacting 99967a88476643f29f948c0d2ad668f0, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=168, earliestPutTs=1732486494877 2024-11-24T22:14:56,788 INFO [RS:0;bd6592c37bb2:40087-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): cdb3383a4019da5f5ef0568dfacd542e#A#compaction#231 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-24T22:14:56,789 INFO [RS:0;bd6592c37bb2:40087-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): cdb3383a4019da5f5ef0568dfacd542e#B#compaction#232 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-24T22:14:56,789 DEBUG [RS:0;bd6592c37bb2:40087-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/cdb3383a4019da5f5ef0568dfacd542e/.tmp/A/ca4d693f159f40c89368d00de5a37289 is 50, key is test_row_0/A:col10/1732486495523/Put/seqid=0 2024-11-24T22:14:56,789 DEBUG [RS:0;bd6592c37bb2:40087-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/cdb3383a4019da5f5ef0568dfacd542e/.tmp/B/3dc8e173764c4c76ba585c4beebbf519 is 50, key is test_row_0/B:col10/1732486495523/Put/seqid=0 2024-11-24T22:14:56,802 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45929 is added to blk_1073742105_1281 (size=12561) 2024-11-24T22:14:56,803 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45929 is added to blk_1073742106_1282 (size=12561) 2024-11-24T22:14:56,808 DEBUG [RS:0;bd6592c37bb2:40087-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/cdb3383a4019da5f5ef0568dfacd542e/.tmp/A/ca4d693f159f40c89368d00de5a37289 as hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/cdb3383a4019da5f5ef0568dfacd542e/A/ca4d693f159f40c89368d00de5a37289 2024-11-24T22:14:56,813 INFO [RS:0;bd6592c37bb2:40087-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in cdb3383a4019da5f5ef0568dfacd542e/A of cdb3383a4019da5f5ef0568dfacd542e into ca4d693f159f40c89368d00de5a37289(size=12.3 K), total size for store is 12.3 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-24T22:14:56,813 DEBUG [RS:0;bd6592c37bb2:40087-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for cdb3383a4019da5f5ef0568dfacd542e: 2024-11-24T22:14:56,813 INFO [RS:0;bd6592c37bb2:40087-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732486488674.cdb3383a4019da5f5ef0568dfacd542e., storeName=cdb3383a4019da5f5ef0568dfacd542e/A, priority=13, startTime=1732486496777; duration=0sec 2024-11-24T22:14:56,813 DEBUG [RS:0;bd6592c37bb2:40087-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-24T22:14:56,813 DEBUG [RS:0;bd6592c37bb2:40087-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: cdb3383a4019da5f5ef0568dfacd542e:A 2024-11-24T22:14:56,813 DEBUG [RS:0;bd6592c37bb2:40087-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-24T22:14:56,814 DEBUG [RS:0;bd6592c37bb2:40087-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36661 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-24T22:14:56,814 DEBUG [RS:0;bd6592c37bb2:40087-shortCompactions-0 {}] regionserver.HStore(1541): cdb3383a4019da5f5ef0568dfacd542e/C is initiating minor compaction (all files) 2024-11-24T22:14:56,814 INFO [RS:0;bd6592c37bb2:40087-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of cdb3383a4019da5f5ef0568dfacd542e/C in TestAcidGuarantees,,1732486488674.cdb3383a4019da5f5ef0568dfacd542e. 2024-11-24T22:14:56,815 INFO [RS:0;bd6592c37bb2:40087-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/cdb3383a4019da5f5ef0568dfacd542e/C/b73271238a90407aae9295d57f98608a, hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/cdb3383a4019da5f5ef0568dfacd542e/C/062cecfaead74cc6864f360d41e8e99f, hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/cdb3383a4019da5f5ef0568dfacd542e/C/6408a0d1112d4fa9853e63edbdd01c09] into tmpdir=hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/cdb3383a4019da5f5ef0568dfacd542e/.tmp, totalSize=35.8 K 2024-11-24T22:14:56,815 DEBUG [RS:0;bd6592c37bb2:40087-shortCompactions-0 {}] compactions.Compactor(225): Compacting b73271238a90407aae9295d57f98608a, keycount=150, bloomtype=ROW, size=12.1 K, encoding=NONE, compression=NONE, seqNum=132, earliestPutTs=1732486492946 2024-11-24T22:14:56,815 DEBUG [RS:0;bd6592c37bb2:40087-shortCompactions-0 {}] compactions.Compactor(225): Compacting 062cecfaead74cc6864f360d41e8e99f, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=149, earliestPutTs=1732486493668 2024-11-24T22:14:56,816 DEBUG [RS:0;bd6592c37bb2:40087-shortCompactions-0 {}] compactions.Compactor(225): Compacting 6408a0d1112d4fa9853e63edbdd01c09, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=168, earliestPutTs=1732486494877 2024-11-24T22:14:56,824 INFO [RS:0;bd6592c37bb2:40087-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): cdb3383a4019da5f5ef0568dfacd542e#C#compaction#233 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-24T22:14:56,826 DEBUG [RS:0;bd6592c37bb2:40087-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/cdb3383a4019da5f5ef0568dfacd542e/.tmp/C/a9bbb3f3509d406a976860c0c208d7c3 is 50, key is test_row_0/C:col10/1732486495523/Put/seqid=0 2024-11-24T22:14:56,838 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45929 is added to blk_1073742107_1283 (size=12561) 2024-11-24T22:14:56,843 DEBUG [RS:0;bd6592c37bb2:40087-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/cdb3383a4019da5f5ef0568dfacd542e/.tmp/C/a9bbb3f3509d406a976860c0c208d7c3 as hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/cdb3383a4019da5f5ef0568dfacd542e/C/a9bbb3f3509d406a976860c0c208d7c3 2024-11-24T22:14:56,848 INFO [RS:0;bd6592c37bb2:40087-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in cdb3383a4019da5f5ef0568dfacd542e/C of cdb3383a4019da5f5ef0568dfacd542e into a9bbb3f3509d406a976860c0c208d7c3(size=12.3 K), total size for store is 12.3 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-24T22:14:56,849 DEBUG [RS:0;bd6592c37bb2:40087-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for cdb3383a4019da5f5ef0568dfacd542e: 2024-11-24T22:14:56,849 INFO [RS:0;bd6592c37bb2:40087-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732486488674.cdb3383a4019da5f5ef0568dfacd542e., storeName=cdb3383a4019da5f5ef0568dfacd542e/C, priority=13, startTime=1732486496777; duration=0sec 2024-11-24T22:14:56,849 DEBUG [RS:0;bd6592c37bb2:40087-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-24T22:14:56,849 DEBUG [RS:0;bd6592c37bb2:40087-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: cdb3383a4019da5f5ef0568dfacd542e:C 2024-11-24T22:14:56,877 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=40087 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=67 2024-11-24T22:14:56,877 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bd6592c37bb2:0-1 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732486488674.cdb3383a4019da5f5ef0568dfacd542e. 2024-11-24T22:14:56,877 INFO [RS_FLUSH_OPERATIONS-regionserver/bd6592c37bb2:0-1 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.HRegion(2902): Flushing cdb3383a4019da5f5ef0568dfacd542e 3/3 column families, dataSize=73.80 KB heapSize=219.89 KB 2024-11-24T22:14:56,877 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bd6592c37bb2:0-1 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.CompactingMemStore(205): FLUSHING TO DISK cdb3383a4019da5f5ef0568dfacd542e, store=A 2024-11-24T22:14:56,877 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bd6592c37bb2:0-1 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-24T22:14:56,877 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bd6592c37bb2:0-1 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.CompactingMemStore(205): FLUSHING TO DISK cdb3383a4019da5f5ef0568dfacd542e, store=B 2024-11-24T22:14:56,877 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bd6592c37bb2:0-1 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-24T22:14:56,877 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bd6592c37bb2:0-1 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.CompactingMemStore(205): FLUSHING TO DISK cdb3383a4019da5f5ef0568dfacd542e, store=C 2024-11-24T22:14:56,877 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bd6592c37bb2:0-1 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-24T22:14:56,882 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bd6592c37bb2:0-1 {event_type=RS_FLUSH_REGIONS, pid=67}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/cdb3383a4019da5f5ef0568dfacd542e/.tmp/A/acf5a3967a5f4ec6b719f46f93df16f7 is 50, key is test_row_0/A:col10/1732486495531/Put/seqid=0 2024-11-24T22:14:56,906 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45929 is added to blk_1073742108_1284 (size=12151) 2024-11-24T22:14:56,907 INFO [RS_FLUSH_OPERATIONS-regionserver/bd6592c37bb2:0-1 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=24.60 KB at sequenceid=184 (bloomFilter=true), to=hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/cdb3383a4019da5f5ef0568dfacd542e/.tmp/A/acf5a3967a5f4ec6b719f46f93df16f7 2024-11-24T22:14:56,916 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bd6592c37bb2:0-1 {event_type=RS_FLUSH_REGIONS, pid=67}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/cdb3383a4019da5f5ef0568dfacd542e/.tmp/B/e9698a5597854a3380c5735bdb28edab is 50, key is test_row_0/B:col10/1732486495531/Put/seqid=0 2024-11-24T22:14:56,935 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45929 is added to blk_1073742109_1285 (size=12151) 2024-11-24T22:14:56,936 INFO [RS_FLUSH_OPERATIONS-regionserver/bd6592c37bb2:0-1 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=24.60 KB at sequenceid=184 (bloomFilter=true), to=hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/cdb3383a4019da5f5ef0568dfacd542e/.tmp/B/e9698a5597854a3380c5735bdb28edab 2024-11-24T22:14:56,941 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37701 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=66 2024-11-24T22:14:56,946 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bd6592c37bb2:0-1 {event_type=RS_FLUSH_REGIONS, pid=67}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/cdb3383a4019da5f5ef0568dfacd542e/.tmp/C/cf79c39e11294570bfd18dc6ef2c4108 is 50, key is test_row_0/C:col10/1732486495531/Put/seqid=0 2024-11-24T22:14:56,958 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45929 is added to blk_1073742110_1286 (size=12151) 2024-11-24T22:14:56,958 INFO [RS_FLUSH_OPERATIONS-regionserver/bd6592c37bb2:0-1 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=24.60 KB at sequenceid=184 (bloomFilter=true), to=hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/cdb3383a4019da5f5ef0568dfacd542e/.tmp/C/cf79c39e11294570bfd18dc6ef2c4108 2024-11-24T22:14:56,965 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bd6592c37bb2:0-1 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/cdb3383a4019da5f5ef0568dfacd542e/.tmp/A/acf5a3967a5f4ec6b719f46f93df16f7 as hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/cdb3383a4019da5f5ef0568dfacd542e/A/acf5a3967a5f4ec6b719f46f93df16f7 2024-11-24T22:14:56,969 INFO [RS_FLUSH_OPERATIONS-regionserver/bd6592c37bb2:0-1 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/cdb3383a4019da5f5ef0568dfacd542e/A/acf5a3967a5f4ec6b719f46f93df16f7, entries=150, sequenceid=184, filesize=11.9 K 2024-11-24T22:14:56,970 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bd6592c37bb2:0-1 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/cdb3383a4019da5f5ef0568dfacd542e/.tmp/B/e9698a5597854a3380c5735bdb28edab as hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/cdb3383a4019da5f5ef0568dfacd542e/B/e9698a5597854a3380c5735bdb28edab 2024-11-24T22:14:56,975 INFO [RS_FLUSH_OPERATIONS-regionserver/bd6592c37bb2:0-1 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/cdb3383a4019da5f5ef0568dfacd542e/B/e9698a5597854a3380c5735bdb28edab, entries=150, sequenceid=184, filesize=11.9 K 2024-11-24T22:14:56,976 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bd6592c37bb2:0-1 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/cdb3383a4019da5f5ef0568dfacd542e/.tmp/C/cf79c39e11294570bfd18dc6ef2c4108 as hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/cdb3383a4019da5f5ef0568dfacd542e/C/cf79c39e11294570bfd18dc6ef2c4108 2024-11-24T22:14:56,980 INFO [RS_FLUSH_OPERATIONS-regionserver/bd6592c37bb2:0-1 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/cdb3383a4019da5f5ef0568dfacd542e/C/cf79c39e11294570bfd18dc6ef2c4108, entries=150, sequenceid=184, filesize=11.9 K 2024-11-24T22:14:56,981 INFO [RS_FLUSH_OPERATIONS-regionserver/bd6592c37bb2:0-1 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.HRegion(3140): Finished flush of dataSize ~73.80 KB/75570, heapSize ~219.84 KB/225120, currentSize=0 B/0 for cdb3383a4019da5f5ef0568dfacd542e in 104ms, sequenceid=184, compaction requested=false 2024-11-24T22:14:56,981 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bd6592c37bb2:0-1 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.HRegion(2603): Flush status journal for cdb3383a4019da5f5ef0568dfacd542e: 2024-11-24T22:14:56,981 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bd6592c37bb2:0-1 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732486488674.cdb3383a4019da5f5ef0568dfacd542e. 2024-11-24T22:14:56,981 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bd6592c37bb2:0-1 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=67 2024-11-24T22:14:56,982 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37701 {}] master.HMaster(4169): Remote procedure done, pid=67 2024-11-24T22:14:56,985 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=67, resume processing ppid=66 2024-11-24T22:14:56,986 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=67, ppid=66, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.1780 sec 2024-11-24T22:14:56,989 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=66, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=66, table=TestAcidGuarantees in 1.1830 sec 2024-11-24T22:14:57,209 DEBUG [RS:0;bd6592c37bb2:40087-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/cdb3383a4019da5f5ef0568dfacd542e/.tmp/B/3dc8e173764c4c76ba585c4beebbf519 as hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/cdb3383a4019da5f5ef0568dfacd542e/B/3dc8e173764c4c76ba585c4beebbf519 2024-11-24T22:14:57,215 INFO [RS:0;bd6592c37bb2:40087-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in cdb3383a4019da5f5ef0568dfacd542e/B of cdb3383a4019da5f5ef0568dfacd542e into 3dc8e173764c4c76ba585c4beebbf519(size=12.3 K), total size for store is 24.1 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-24T22:14:57,215 DEBUG [RS:0;bd6592c37bb2:40087-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for cdb3383a4019da5f5ef0568dfacd542e: 2024-11-24T22:14:57,215 INFO [RS:0;bd6592c37bb2:40087-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732486488674.cdb3383a4019da5f5ef0568dfacd542e., storeName=cdb3383a4019da5f5ef0568dfacd542e/B, priority=13, startTime=1732486496777; duration=0sec 2024-11-24T22:14:57,216 DEBUG [RS:0;bd6592c37bb2:40087-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-24T22:14:57,216 DEBUG [RS:0;bd6592c37bb2:40087-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: cdb3383a4019da5f5ef0568dfacd542e:B 2024-11-24T22:14:57,745 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40087 {}] regionserver.HRegion(8855): Flush requested on cdb3383a4019da5f5ef0568dfacd542e 2024-11-24T22:14:57,745 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing cdb3383a4019da5f5ef0568dfacd542e 3/3 column families, dataSize=46.96 KB heapSize=140.20 KB 2024-11-24T22:14:57,746 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK cdb3383a4019da5f5ef0568dfacd542e, store=A 2024-11-24T22:14:57,746 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-24T22:14:57,746 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK cdb3383a4019da5f5ef0568dfacd542e, store=B 2024-11-24T22:14:57,746 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-24T22:14:57,746 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK cdb3383a4019da5f5ef0568dfacd542e, store=C 2024-11-24T22:14:57,746 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-24T22:14:57,755 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/cdb3383a4019da5f5ef0568dfacd542e/.tmp/A/6a5083c0ee774458bf24c31fed15ca5c is 50, key is test_row_0/A:col10/1732486497740/Put/seqid=0 2024-11-24T22:14:57,759 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45929 is added to blk_1073742111_1287 (size=14541) 2024-11-24T22:14:57,836 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/bd6592c37bb2:0-6 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1732486497771, max timestamp=1732486497832Num uniques 859; 2024-11-24T22:14:57,836 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/bd6592c37bb2:0-8 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1732486497771, max timestamp=1732486497832Num uniques 859; 2024-11-24T22:14:57,836 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/bd6592c37bb2:0-9 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1732486497771, max timestamp=1732486497832Num uniques 859; 2024-11-24T22:14:57,836 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/bd6592c37bb2:0-9 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-11-24T22:14:57,836 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/bd6592c37bb2:0-8 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-11-24T22:14:57,836 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/bd6592c37bb2:0-6 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-11-24T22:14:57,938 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40087 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cdb3383a4019da5f5ef0568dfacd542e, server=bd6592c37bb2,40087,1732486423713 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-24T22:14:57,938 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40087 {}] ipc.CallRunner(138): callId: 170 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40834 deadline: 1732486557933, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cdb3383a4019da5f5ef0568dfacd542e, server=bd6592c37bb2,40087,1732486423713 2024-11-24T22:14:57,938 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732486488674.cdb3383a4019da5f5ef0568dfacd542e., hostname=bd6592c37bb2,40087,1732486423713, seqNum=2 , the old value is region=TestAcidGuarantees,,1732486488674.cdb3383a4019da5f5ef0568dfacd542e., hostname=bd6592c37bb2,40087,1732486423713, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cdb3383a4019da5f5ef0568dfacd542e, server=bd6592c37bb2,40087,1732486423713 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T22:14:57,939 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732486488674.cdb3383a4019da5f5ef0568dfacd542e., hostname=bd6592c37bb2,40087,1732486423713, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cdb3383a4019da5f5ef0568dfacd542e, server=bd6592c37bb2,40087,1732486423713 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T22:14:57,939 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732486488674.cdb3383a4019da5f5ef0568dfacd542e., hostname=bd6592c37bb2,40087,1732486423713, seqNum=2 because the exception is null or not the one we care about 2024-11-24T22:14:57,951 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37701 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=66 2024-11-24T22:14:57,951 INFO [RPCClient-NioEventLoopGroup-4-6 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-24T22:14:57,951 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37701 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-24T22:14:57,953 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37701 {}] procedure2.ProcedureExecutor(1139): Stored pid=68, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=68, table=TestAcidGuarantees 2024-11-24T22:14:57,953 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37701 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=68 2024-11-24T22:14:57,954 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=68, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=68, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-24T22:14:57,954 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=68, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=68, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-24T22:14:57,955 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=69, ppid=68, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-24T22:14:58,044 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40087 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cdb3383a4019da5f5ef0568dfacd542e, server=bd6592c37bb2,40087,1732486423713 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-24T22:14:58,044 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40087 {}] ipc.CallRunner(138): callId: 171 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40834 deadline: 1732486558040, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cdb3383a4019da5f5ef0568dfacd542e, server=bd6592c37bb2,40087,1732486423713 2024-11-24T22:14:58,044 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732486488674.cdb3383a4019da5f5ef0568dfacd542e., hostname=bd6592c37bb2,40087,1732486423713, seqNum=2 , the old value is region=TestAcidGuarantees,,1732486488674.cdb3383a4019da5f5ef0568dfacd542e., hostname=bd6592c37bb2,40087,1732486423713, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cdb3383a4019da5f5ef0568dfacd542e, server=bd6592c37bb2,40087,1732486423713 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T22:14:58,045 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732486488674.cdb3383a4019da5f5ef0568dfacd542e., hostname=bd6592c37bb2,40087,1732486423713, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cdb3383a4019da5f5ef0568dfacd542e, server=bd6592c37bb2,40087,1732486423713 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T22:14:58,045 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732486488674.cdb3383a4019da5f5ef0568dfacd542e., hostname=bd6592c37bb2,40087,1732486423713, seqNum=2 because the exception is null or not the one we care about 2024-11-24T22:14:58,061 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37701 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=68 2024-11-24T22:14:58,108 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=40087 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=69 2024-11-24T22:14:58,108 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bd6592c37bb2:0-2 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732486488674.cdb3383a4019da5f5ef0568dfacd542e. 2024-11-24T22:14:58,108 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bd6592c37bb2:0-2 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732486488674.cdb3383a4019da5f5ef0568dfacd542e. as already flushing 2024-11-24T22:14:58,108 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bd6592c37bb2:0-2 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732486488674.cdb3383a4019da5f5ef0568dfacd542e. 2024-11-24T22:14:58,108 ERROR [RS_FLUSH_OPERATIONS-regionserver/bd6592c37bb2:0-2 {event_type=RS_FLUSH_REGIONS, pid=69}] handler.RSProcedureHandler(58): pid=69 java.io.IOException: Unable to complete flush {ENCODED => cdb3383a4019da5f5ef0568dfacd542e, NAME => 'TestAcidGuarantees,,1732486488674.cdb3383a4019da5f5ef0568dfacd542e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T22:14:58,108 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bd6592c37bb2:0-2 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=69 java.io.IOException: Unable to complete flush {ENCODED => cdb3383a4019da5f5ef0568dfacd542e, NAME => 'TestAcidGuarantees,,1732486488674.cdb3383a4019da5f5ef0568dfacd542e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T22:14:58,109 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37701 {}] master.HMaster(4177): Remote procedure failed, pid=69 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => cdb3383a4019da5f5ef0568dfacd542e, NAME => 'TestAcidGuarantees,,1732486488674.cdb3383a4019da5f5ef0568dfacd542e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => cdb3383a4019da5f5ef0568dfacd542e, NAME => 'TestAcidGuarantees,,1732486488674.cdb3383a4019da5f5ef0568dfacd542e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T22:14:58,160 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=195 (bloomFilter=true), to=hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/cdb3383a4019da5f5ef0568dfacd542e/.tmp/A/6a5083c0ee774458bf24c31fed15ca5c 2024-11-24T22:14:58,169 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/cdb3383a4019da5f5ef0568dfacd542e/.tmp/B/67c2d2bcd6e944d1a80d70b92d26f09e is 50, key is test_row_0/B:col10/1732486497740/Put/seqid=0 2024-11-24T22:14:58,172 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45929 is added to blk_1073742112_1288 (size=12151) 2024-11-24T22:14:58,172 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=195 (bloomFilter=true), to=hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/cdb3383a4019da5f5ef0568dfacd542e/.tmp/B/67c2d2bcd6e944d1a80d70b92d26f09e 2024-11-24T22:14:58,179 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/cdb3383a4019da5f5ef0568dfacd542e/.tmp/C/c112743a0d3747e89caee36a9f12cf6f is 50, key is test_row_0/C:col10/1732486497740/Put/seqid=0 2024-11-24T22:14:58,182 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45929 is added to blk_1073742113_1289 (size=12151) 2024-11-24T22:14:58,253 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40087 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cdb3383a4019da5f5ef0568dfacd542e, server=bd6592c37bb2,40087,1732486423713 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-24T22:14:58,253 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40087 {}] ipc.CallRunner(138): callId: 172 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40834 deadline: 1732486558251, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cdb3383a4019da5f5ef0568dfacd542e, server=bd6592c37bb2,40087,1732486423713 2024-11-24T22:14:58,253 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732486488674.cdb3383a4019da5f5ef0568dfacd542e., hostname=bd6592c37bb2,40087,1732486423713, seqNum=2 , the old value is region=TestAcidGuarantees,,1732486488674.cdb3383a4019da5f5ef0568dfacd542e., hostname=bd6592c37bb2,40087,1732486423713, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cdb3383a4019da5f5ef0568dfacd542e, server=bd6592c37bb2,40087,1732486423713 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T22:14:58,253 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732486488674.cdb3383a4019da5f5ef0568dfacd542e., hostname=bd6592c37bb2,40087,1732486423713, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cdb3383a4019da5f5ef0568dfacd542e, server=bd6592c37bb2,40087,1732486423713 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T22:14:58,253 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732486488674.cdb3383a4019da5f5ef0568dfacd542e., hostname=bd6592c37bb2,40087,1732486423713, seqNum=2 because the exception is null or not the one we care about 2024-11-24T22:14:58,261 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=40087 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=69 2024-11-24T22:14:58,261 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bd6592c37bb2:0-0 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732486488674.cdb3383a4019da5f5ef0568dfacd542e. 2024-11-24T22:14:58,261 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bd6592c37bb2:0-0 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732486488674.cdb3383a4019da5f5ef0568dfacd542e. as already flushing 2024-11-24T22:14:58,261 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bd6592c37bb2:0-0 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732486488674.cdb3383a4019da5f5ef0568dfacd542e. 2024-11-24T22:14:58,261 ERROR [RS_FLUSH_OPERATIONS-regionserver/bd6592c37bb2:0-0 {event_type=RS_FLUSH_REGIONS, pid=69}] handler.RSProcedureHandler(58): pid=69 java.io.IOException: Unable to complete flush {ENCODED => cdb3383a4019da5f5ef0568dfacd542e, NAME => 'TestAcidGuarantees,,1732486488674.cdb3383a4019da5f5ef0568dfacd542e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T22:14:58,261 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bd6592c37bb2:0-0 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=69 java.io.IOException: Unable to complete flush {ENCODED => cdb3383a4019da5f5ef0568dfacd542e, NAME => 'TestAcidGuarantees,,1732486488674.cdb3383a4019da5f5ef0568dfacd542e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T22:14:58,262 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37701 {}] master.HMaster(4177): Remote procedure failed, pid=69 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => cdb3383a4019da5f5ef0568dfacd542e, NAME => 'TestAcidGuarantees,,1732486488674.cdb3383a4019da5f5ef0568dfacd542e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => cdb3383a4019da5f5ef0568dfacd542e, NAME => 'TestAcidGuarantees,,1732486488674.cdb3383a4019da5f5ef0568dfacd542e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T22:14:58,270 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37701 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=68 2024-11-24T22:14:58,414 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=40087 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=69 2024-11-24T22:14:58,414 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bd6592c37bb2:0-1 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732486488674.cdb3383a4019da5f5ef0568dfacd542e. 2024-11-24T22:14:58,414 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bd6592c37bb2:0-1 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732486488674.cdb3383a4019da5f5ef0568dfacd542e. as already flushing 2024-11-24T22:14:58,414 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bd6592c37bb2:0-1 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732486488674.cdb3383a4019da5f5ef0568dfacd542e. 2024-11-24T22:14:58,414 ERROR [RS_FLUSH_OPERATIONS-regionserver/bd6592c37bb2:0-1 {event_type=RS_FLUSH_REGIONS, pid=69}] handler.RSProcedureHandler(58): pid=69 java.io.IOException: Unable to complete flush {ENCODED => cdb3383a4019da5f5ef0568dfacd542e, NAME => 'TestAcidGuarantees,,1732486488674.cdb3383a4019da5f5ef0568dfacd542e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T22:14:58,414 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bd6592c37bb2:0-1 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=69 java.io.IOException: Unable to complete flush {ENCODED => cdb3383a4019da5f5ef0568dfacd542e, NAME => 'TestAcidGuarantees,,1732486488674.cdb3383a4019da5f5ef0568dfacd542e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T22:14:58,415 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37701 {}] master.HMaster(4177): Remote procedure failed, pid=69 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => cdb3383a4019da5f5ef0568dfacd542e, NAME => 'TestAcidGuarantees,,1732486488674.cdb3383a4019da5f5ef0568dfacd542e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => cdb3383a4019da5f5ef0568dfacd542e, NAME => 'TestAcidGuarantees,,1732486488674.cdb3383a4019da5f5ef0568dfacd542e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T22:14:58,566 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40087 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cdb3383a4019da5f5ef0568dfacd542e, server=bd6592c37bb2,40087,1732486423713 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-24T22:14:58,566 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40087 {}] ipc.CallRunner(138): callId: 173 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40834 deadline: 1732486558561, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cdb3383a4019da5f5ef0568dfacd542e, server=bd6592c37bb2,40087,1732486423713 2024-11-24T22:14:58,567 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732486488674.cdb3383a4019da5f5ef0568dfacd542e., hostname=bd6592c37bb2,40087,1732486423713, seqNum=2 , the old value is region=TestAcidGuarantees,,1732486488674.cdb3383a4019da5f5ef0568dfacd542e., hostname=bd6592c37bb2,40087,1732486423713, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cdb3383a4019da5f5ef0568dfacd542e, server=bd6592c37bb2,40087,1732486423713 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T22:14:58,567 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732486488674.cdb3383a4019da5f5ef0568dfacd542e., hostname=bd6592c37bb2,40087,1732486423713, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cdb3383a4019da5f5ef0568dfacd542e, server=bd6592c37bb2,40087,1732486423713 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T22:14:58,567 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732486488674.cdb3383a4019da5f5ef0568dfacd542e., hostname=bd6592c37bb2,40087,1732486423713, seqNum=2 because the exception is null or not the one we care about 2024-11-24T22:14:58,567 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=40087 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=69 2024-11-24T22:14:58,567 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bd6592c37bb2:0-2 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732486488674.cdb3383a4019da5f5ef0568dfacd542e. 2024-11-24T22:14:58,567 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bd6592c37bb2:0-2 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732486488674.cdb3383a4019da5f5ef0568dfacd542e. as already flushing 2024-11-24T22:14:58,567 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bd6592c37bb2:0-2 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732486488674.cdb3383a4019da5f5ef0568dfacd542e. 2024-11-24T22:14:58,567 ERROR [RS_FLUSH_OPERATIONS-regionserver/bd6592c37bb2:0-2 {event_type=RS_FLUSH_REGIONS, pid=69}] handler.RSProcedureHandler(58): pid=69 java.io.IOException: Unable to complete flush {ENCODED => cdb3383a4019da5f5ef0568dfacd542e, NAME => 'TestAcidGuarantees,,1732486488674.cdb3383a4019da5f5ef0568dfacd542e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T22:14:58,568 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bd6592c37bb2:0-2 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=69 java.io.IOException: Unable to complete flush {ENCODED => cdb3383a4019da5f5ef0568dfacd542e, NAME => 'TestAcidGuarantees,,1732486488674.cdb3383a4019da5f5ef0568dfacd542e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T22:14:58,568 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37701 {}] master.HMaster(4177): Remote procedure failed, pid=69 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => cdb3383a4019da5f5ef0568dfacd542e, NAME => 'TestAcidGuarantees,,1732486488674.cdb3383a4019da5f5ef0568dfacd542e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => cdb3383a4019da5f5ef0568dfacd542e, NAME => 'TestAcidGuarantees,,1732486488674.cdb3383a4019da5f5ef0568dfacd542e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T22:14:58,581 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37701 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=68 2024-11-24T22:14:58,583 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=195 (bloomFilter=true), to=hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/cdb3383a4019da5f5ef0568dfacd542e/.tmp/C/c112743a0d3747e89caee36a9f12cf6f 2024-11-24T22:14:58,587 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/cdb3383a4019da5f5ef0568dfacd542e/.tmp/A/6a5083c0ee774458bf24c31fed15ca5c as hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/cdb3383a4019da5f5ef0568dfacd542e/A/6a5083c0ee774458bf24c31fed15ca5c 2024-11-24T22:14:58,591 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/cdb3383a4019da5f5ef0568dfacd542e/A/6a5083c0ee774458bf24c31fed15ca5c, entries=200, sequenceid=195, filesize=14.2 K 2024-11-24T22:14:58,592 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/cdb3383a4019da5f5ef0568dfacd542e/.tmp/B/67c2d2bcd6e944d1a80d70b92d26f09e as hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/cdb3383a4019da5f5ef0568dfacd542e/B/67c2d2bcd6e944d1a80d70b92d26f09e 2024-11-24T22:14:58,595 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/cdb3383a4019da5f5ef0568dfacd542e/B/67c2d2bcd6e944d1a80d70b92d26f09e, entries=150, sequenceid=195, filesize=11.9 K 2024-11-24T22:14:58,596 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/cdb3383a4019da5f5ef0568dfacd542e/.tmp/C/c112743a0d3747e89caee36a9f12cf6f as hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/cdb3383a4019da5f5ef0568dfacd542e/C/c112743a0d3747e89caee36a9f12cf6f 2024-11-24T22:14:58,599 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/cdb3383a4019da5f5ef0568dfacd542e/C/c112743a0d3747e89caee36a9f12cf6f, entries=150, sequenceid=195, filesize=11.9 K 2024-11-24T22:14:58,600 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~46.96 KB/48090, heapSize ~140.16 KB/143520, currentSize=134.18 KB/137400 for cdb3383a4019da5f5ef0568dfacd542e in 855ms, sequenceid=195, compaction requested=true 2024-11-24T22:14:58,600 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for cdb3383a4019da5f5ef0568dfacd542e: 2024-11-24T22:14:58,600 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store cdb3383a4019da5f5ef0568dfacd542e:A, priority=-2147483648, current under compaction store size is 1 2024-11-24T22:14:58,600 DEBUG [RS:0;bd6592c37bb2:40087-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-24T22:14:58,600 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-24T22:14:58,600 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store cdb3383a4019da5f5ef0568dfacd542e:B, priority=-2147483648, current under compaction store size is 2 2024-11-24T22:14:58,600 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-24T22:14:58,600 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store cdb3383a4019da5f5ef0568dfacd542e:C, priority=-2147483648, current under compaction store size is 3 2024-11-24T22:14:58,600 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-24T22:14:58,600 DEBUG [RS:0;bd6592c37bb2:40087-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-24T22:14:58,601 DEBUG [RS:0;bd6592c37bb2:40087-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36863 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-24T22:14:58,601 DEBUG [RS:0;bd6592c37bb2:40087-longCompactions-0 {}] regionserver.HStore(1541): cdb3383a4019da5f5ef0568dfacd542e/B is initiating minor compaction (all files) 2024-11-24T22:14:58,601 DEBUG [RS:0;bd6592c37bb2:40087-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 39253 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-24T22:14:58,601 INFO [RS:0;bd6592c37bb2:40087-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of cdb3383a4019da5f5ef0568dfacd542e/B in TestAcidGuarantees,,1732486488674.cdb3383a4019da5f5ef0568dfacd542e. 2024-11-24T22:14:58,601 DEBUG [RS:0;bd6592c37bb2:40087-shortCompactions-0 {}] regionserver.HStore(1541): cdb3383a4019da5f5ef0568dfacd542e/A is initiating minor compaction (all files) 2024-11-24T22:14:58,601 INFO [RS:0;bd6592c37bb2:40087-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of cdb3383a4019da5f5ef0568dfacd542e/A in TestAcidGuarantees,,1732486488674.cdb3383a4019da5f5ef0568dfacd542e. 2024-11-24T22:14:58,601 INFO [RS:0;bd6592c37bb2:40087-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/cdb3383a4019da5f5ef0568dfacd542e/B/3dc8e173764c4c76ba585c4beebbf519, hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/cdb3383a4019da5f5ef0568dfacd542e/B/e9698a5597854a3380c5735bdb28edab, hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/cdb3383a4019da5f5ef0568dfacd542e/B/67c2d2bcd6e944d1a80d70b92d26f09e] into tmpdir=hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/cdb3383a4019da5f5ef0568dfacd542e/.tmp, totalSize=36.0 K 2024-11-24T22:14:58,601 INFO [RS:0;bd6592c37bb2:40087-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/cdb3383a4019da5f5ef0568dfacd542e/A/ca4d693f159f40c89368d00de5a37289, hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/cdb3383a4019da5f5ef0568dfacd542e/A/acf5a3967a5f4ec6b719f46f93df16f7, hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/cdb3383a4019da5f5ef0568dfacd542e/A/6a5083c0ee774458bf24c31fed15ca5c] into tmpdir=hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/cdb3383a4019da5f5ef0568dfacd542e/.tmp, totalSize=38.3 K 2024-11-24T22:14:58,602 DEBUG [RS:0;bd6592c37bb2:40087-longCompactions-0 {}] compactions.Compactor(225): Compacting 3dc8e173764c4c76ba585c4beebbf519, keycount=150, bloomtype=ROW, size=12.3 K, encoding=NONE, compression=NONE, seqNum=168, earliestPutTs=1732486494877 2024-11-24T22:14:58,602 DEBUG [RS:0;bd6592c37bb2:40087-shortCompactions-0 {}] compactions.Compactor(225): Compacting ca4d693f159f40c89368d00de5a37289, keycount=150, bloomtype=ROW, size=12.3 K, encoding=NONE, compression=NONE, seqNum=168, earliestPutTs=1732486494877 2024-11-24T22:14:58,602 DEBUG [RS:0;bd6592c37bb2:40087-longCompactions-0 {}] compactions.Compactor(225): Compacting e9698a5597854a3380c5735bdb28edab, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=184, earliestPutTs=1732486495531 2024-11-24T22:14:58,602 DEBUG [RS:0;bd6592c37bb2:40087-shortCompactions-0 {}] compactions.Compactor(225): Compacting acf5a3967a5f4ec6b719f46f93df16f7, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=184, earliestPutTs=1732486495531 2024-11-24T22:14:58,602 DEBUG [RS:0;bd6592c37bb2:40087-longCompactions-0 {}] compactions.Compactor(225): Compacting 67c2d2bcd6e944d1a80d70b92d26f09e, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=195, earliestPutTs=1732486497733 2024-11-24T22:14:58,603 DEBUG [RS:0;bd6592c37bb2:40087-shortCompactions-0 {}] compactions.Compactor(225): Compacting 6a5083c0ee774458bf24c31fed15ca5c, keycount=200, bloomtype=ROW, size=14.2 K, encoding=NONE, compression=NONE, seqNum=195, earliestPutTs=1732486497733 2024-11-24T22:14:58,610 INFO [RS:0;bd6592c37bb2:40087-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): cdb3383a4019da5f5ef0568dfacd542e#B#compaction#241 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-24T22:14:58,610 INFO [RS:0;bd6592c37bb2:40087-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): cdb3383a4019da5f5ef0568dfacd542e#A#compaction#240 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-24T22:14:58,610 DEBUG [RS:0;bd6592c37bb2:40087-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/cdb3383a4019da5f5ef0568dfacd542e/.tmp/B/e94ce2756c6743089e95d153fcda9749 is 50, key is test_row_0/B:col10/1732486497740/Put/seqid=0 2024-11-24T22:14:58,611 DEBUG [RS:0;bd6592c37bb2:40087-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/cdb3383a4019da5f5ef0568dfacd542e/.tmp/A/79b97ac10a544c658a9ec43f5d94fe7f is 50, key is test_row_0/A:col10/1732486497740/Put/seqid=0 2024-11-24T22:14:58,615 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45929 is added to blk_1073742115_1291 (size=12663) 2024-11-24T22:14:58,617 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45929 is added to blk_1073742114_1290 (size=12663) 2024-11-24T22:14:58,622 DEBUG [RS:0;bd6592c37bb2:40087-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/cdb3383a4019da5f5ef0568dfacd542e/.tmp/A/79b97ac10a544c658a9ec43f5d94fe7f as hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/cdb3383a4019da5f5ef0568dfacd542e/A/79b97ac10a544c658a9ec43f5d94fe7f 2024-11-24T22:14:58,628 INFO [RS:0;bd6592c37bb2:40087-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in cdb3383a4019da5f5ef0568dfacd542e/A of cdb3383a4019da5f5ef0568dfacd542e into 79b97ac10a544c658a9ec43f5d94fe7f(size=12.4 K), total size for store is 12.4 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-24T22:14:58,628 DEBUG [RS:0;bd6592c37bb2:40087-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for cdb3383a4019da5f5ef0568dfacd542e: 2024-11-24T22:14:58,628 INFO [RS:0;bd6592c37bb2:40087-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732486488674.cdb3383a4019da5f5ef0568dfacd542e., storeName=cdb3383a4019da5f5ef0568dfacd542e/A, priority=13, startTime=1732486498600; duration=0sec 2024-11-24T22:14:58,628 DEBUG [RS:0;bd6592c37bb2:40087-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-24T22:14:58,628 DEBUG [RS:0;bd6592c37bb2:40087-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: cdb3383a4019da5f5ef0568dfacd542e:A 2024-11-24T22:14:58,628 DEBUG [RS:0;bd6592c37bb2:40087-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-24T22:14:58,629 DEBUG [RS:0;bd6592c37bb2:40087-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36863 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-24T22:14:58,630 DEBUG [RS:0;bd6592c37bb2:40087-shortCompactions-0 {}] regionserver.HStore(1541): cdb3383a4019da5f5ef0568dfacd542e/C is initiating minor compaction (all files) 2024-11-24T22:14:58,630 INFO [RS:0;bd6592c37bb2:40087-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of cdb3383a4019da5f5ef0568dfacd542e/C in TestAcidGuarantees,,1732486488674.cdb3383a4019da5f5ef0568dfacd542e. 2024-11-24T22:14:58,630 INFO [RS:0;bd6592c37bb2:40087-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/cdb3383a4019da5f5ef0568dfacd542e/C/a9bbb3f3509d406a976860c0c208d7c3, hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/cdb3383a4019da5f5ef0568dfacd542e/C/cf79c39e11294570bfd18dc6ef2c4108, hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/cdb3383a4019da5f5ef0568dfacd542e/C/c112743a0d3747e89caee36a9f12cf6f] into tmpdir=hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/cdb3383a4019da5f5ef0568dfacd542e/.tmp, totalSize=36.0 K 2024-11-24T22:14:58,630 DEBUG [RS:0;bd6592c37bb2:40087-shortCompactions-0 {}] compactions.Compactor(225): Compacting a9bbb3f3509d406a976860c0c208d7c3, keycount=150, bloomtype=ROW, size=12.3 K, encoding=NONE, compression=NONE, seqNum=168, earliestPutTs=1732486494877 2024-11-24T22:14:58,631 DEBUG [RS:0;bd6592c37bb2:40087-shortCompactions-0 {}] compactions.Compactor(225): Compacting cf79c39e11294570bfd18dc6ef2c4108, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=184, earliestPutTs=1732486495531 2024-11-24T22:14:58,631 DEBUG [RS:0;bd6592c37bb2:40087-shortCompactions-0 {}] compactions.Compactor(225): Compacting c112743a0d3747e89caee36a9f12cf6f, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=195, earliestPutTs=1732486497733 2024-11-24T22:14:58,638 INFO [RS:0;bd6592c37bb2:40087-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): cdb3383a4019da5f5ef0568dfacd542e#C#compaction#242 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-24T22:14:58,638 DEBUG [RS:0;bd6592c37bb2:40087-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/cdb3383a4019da5f5ef0568dfacd542e/.tmp/C/6ad8c8648bc049cdb645eb8af4e16679 is 50, key is test_row_0/C:col10/1732486497740/Put/seqid=0 2024-11-24T22:14:58,643 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45929 is added to blk_1073742116_1292 (size=12663) 2024-11-24T22:14:58,720 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=40087 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=69 2024-11-24T22:14:58,720 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bd6592c37bb2:0-0 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732486488674.cdb3383a4019da5f5ef0568dfacd542e. 2024-11-24T22:14:58,720 INFO [RS_FLUSH_OPERATIONS-regionserver/bd6592c37bb2:0-0 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.HRegion(2902): Flushing cdb3383a4019da5f5ef0568dfacd542e 3/3 column families, dataSize=134.18 KB heapSize=384.98 KB 2024-11-24T22:14:58,721 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bd6592c37bb2:0-0 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.CompactingMemStore(205): FLUSHING TO DISK cdb3383a4019da5f5ef0568dfacd542e, store=A 2024-11-24T22:14:58,721 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bd6592c37bb2:0-0 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-24T22:14:58,721 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bd6592c37bb2:0-0 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.CompactingMemStore(205): FLUSHING TO DISK cdb3383a4019da5f5ef0568dfacd542e, store=B 2024-11-24T22:14:58,721 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bd6592c37bb2:0-0 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-24T22:14:58,721 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bd6592c37bb2:0-0 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.CompactingMemStore(205): FLUSHING TO DISK cdb3383a4019da5f5ef0568dfacd542e, store=C 2024-11-24T22:14:58,721 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bd6592c37bb2:0-0 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-24T22:14:58,726 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bd6592c37bb2:0-0 {event_type=RS_FLUSH_REGIONS, pid=69}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/cdb3383a4019da5f5ef0568dfacd542e/.tmp/A/fd12ade02bb949afabf88523cee4fd62 is 50, key is test_row_0/A:col10/1732486497925/Put/seqid=0 2024-11-24T22:14:58,730 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45929 is added to blk_1073742117_1293 (size=12151) 2024-11-24T22:14:59,023 DEBUG [RS:0;bd6592c37bb2:40087-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/cdb3383a4019da5f5ef0568dfacd542e/.tmp/B/e94ce2756c6743089e95d153fcda9749 as hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/cdb3383a4019da5f5ef0568dfacd542e/B/e94ce2756c6743089e95d153fcda9749 2024-11-24T22:14:59,028 INFO [RS:0;bd6592c37bb2:40087-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in cdb3383a4019da5f5ef0568dfacd542e/B of cdb3383a4019da5f5ef0568dfacd542e into e94ce2756c6743089e95d153fcda9749(size=12.4 K), total size for store is 12.4 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-24T22:14:59,028 DEBUG [RS:0;bd6592c37bb2:40087-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for cdb3383a4019da5f5ef0568dfacd542e: 2024-11-24T22:14:59,028 INFO [RS:0;bd6592c37bb2:40087-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732486488674.cdb3383a4019da5f5ef0568dfacd542e., storeName=cdb3383a4019da5f5ef0568dfacd542e/B, priority=13, startTime=1732486498600; duration=0sec 2024-11-24T22:14:59,028 DEBUG [RS:0;bd6592c37bb2:40087-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-24T22:14:59,028 DEBUG [RS:0;bd6592c37bb2:40087-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: cdb3383a4019da5f5ef0568dfacd542e:B 2024-11-24T22:14:59,047 DEBUG [RS:0;bd6592c37bb2:40087-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/cdb3383a4019da5f5ef0568dfacd542e/.tmp/C/6ad8c8648bc049cdb645eb8af4e16679 as hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/cdb3383a4019da5f5ef0568dfacd542e/C/6ad8c8648bc049cdb645eb8af4e16679 2024-11-24T22:14:59,052 INFO [RS:0;bd6592c37bb2:40087-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in cdb3383a4019da5f5ef0568dfacd542e/C of cdb3383a4019da5f5ef0568dfacd542e into 6ad8c8648bc049cdb645eb8af4e16679(size=12.4 K), total size for store is 12.4 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-24T22:14:59,052 DEBUG [RS:0;bd6592c37bb2:40087-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for cdb3383a4019da5f5ef0568dfacd542e: 2024-11-24T22:14:59,052 INFO [RS:0;bd6592c37bb2:40087-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732486488674.cdb3383a4019da5f5ef0568dfacd542e., storeName=cdb3383a4019da5f5ef0568dfacd542e/C, priority=13, startTime=1732486498600; duration=0sec 2024-11-24T22:14:59,052 DEBUG [RS:0;bd6592c37bb2:40087-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-24T22:14:59,052 DEBUG [RS:0;bd6592c37bb2:40087-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: cdb3383a4019da5f5ef0568dfacd542e:C 2024-11-24T22:14:59,074 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40087 {}] regionserver.HRegion(8855): Flush requested on cdb3383a4019da5f5ef0568dfacd542e 2024-11-24T22:14:59,074 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732486488674.cdb3383a4019da5f5ef0568dfacd542e. as already flushing 2024-11-24T22:14:59,090 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37701 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=68 2024-11-24T22:14:59,112 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40087 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cdb3383a4019da5f5ef0568dfacd542e, server=bd6592c37bb2,40087,1732486423713 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-24T22:14:59,112 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40087 {}] ipc.CallRunner(138): callId: 181 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40834 deadline: 1732486559107, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cdb3383a4019da5f5ef0568dfacd542e, server=bd6592c37bb2,40087,1732486423713 2024-11-24T22:14:59,113 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732486488674.cdb3383a4019da5f5ef0568dfacd542e., hostname=bd6592c37bb2,40087,1732486423713, seqNum=2 , the old value is region=TestAcidGuarantees,,1732486488674.cdb3383a4019da5f5ef0568dfacd542e., hostname=bd6592c37bb2,40087,1732486423713, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cdb3383a4019da5f5ef0568dfacd542e, server=bd6592c37bb2,40087,1732486423713 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T22:14:59,113 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732486488674.cdb3383a4019da5f5ef0568dfacd542e., hostname=bd6592c37bb2,40087,1732486423713, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cdb3383a4019da5f5ef0568dfacd542e, server=bd6592c37bb2,40087,1732486423713 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T22:14:59,113 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732486488674.cdb3383a4019da5f5ef0568dfacd542e., hostname=bd6592c37bb2,40087,1732486423713, seqNum=2 because the exception is null or not the one we care about 2024-11-24T22:14:59,131 INFO [RS_FLUSH_OPERATIONS-regionserver/bd6592c37bb2:0-0 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=233 (bloomFilter=true), to=hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/cdb3383a4019da5f5ef0568dfacd542e/.tmp/A/fd12ade02bb949afabf88523cee4fd62 2024-11-24T22:14:59,141 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bd6592c37bb2:0-0 {event_type=RS_FLUSH_REGIONS, pid=69}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/cdb3383a4019da5f5ef0568dfacd542e/.tmp/B/1511cbf759814b93b4166c8ac54f35ca is 50, key is test_row_0/B:col10/1732486497925/Put/seqid=0 2024-11-24T22:14:59,145 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45929 is added to blk_1073742118_1294 (size=12151) 2024-11-24T22:14:59,175 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40087 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cdb3383a4019da5f5ef0568dfacd542e, server=bd6592c37bb2,40087,1732486423713 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-24T22:14:59,175 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40087 {}] ipc.CallRunner(138): callId: 17 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40806 deadline: 1732486559170, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cdb3383a4019da5f5ef0568dfacd542e, server=bd6592c37bb2,40087,1732486423713 2024-11-24T22:14:59,175 WARN [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRpcRetryingCaller(177): Call to bd6592c37bb2,40087,1732486423713 for 'test_row_2' in cdb3383a4019da5f5ef0568dfacd542e of TestAcidGuarantees failed, tries = 8, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 8216 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cdb3383a4019da5f5ef0568dfacd542e, server=bd6592c37bb2,40087,1732486423713 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor44.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cdb3383a4019da5f5ef0568dfacd542e, server=bd6592c37bb2,40087,1732486423713 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-24T22:14:59,176 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732486488674.cdb3383a4019da5f5ef0568dfacd542e., hostname=bd6592c37bb2,40087,1732486423713, seqNum=2 , the old value is region=TestAcidGuarantees,,1732486488674.cdb3383a4019da5f5ef0568dfacd542e., hostname=bd6592c37bb2,40087,1732486423713, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cdb3383a4019da5f5ef0568dfacd542e, server=bd6592c37bb2,40087,1732486423713 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T22:14:59,176 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732486488674.cdb3383a4019da5f5ef0568dfacd542e., hostname=bd6592c37bb2,40087,1732486423713, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cdb3383a4019da5f5ef0568dfacd542e, server=bd6592c37bb2,40087,1732486423713 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T22:14:59,176 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732486488674.cdb3383a4019da5f5ef0568dfacd542e., hostname=bd6592c37bb2,40087,1732486423713, seqNum=2 because the exception is null or not the one we care about 2024-11-24T22:14:59,184 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40087 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cdb3383a4019da5f5ef0568dfacd542e, server=bd6592c37bb2,40087,1732486423713 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-24T22:14:59,185 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40087 {}] ipc.CallRunner(138): callId: 17 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40816 deadline: 1732486559181, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cdb3383a4019da5f5ef0568dfacd542e, server=bd6592c37bb2,40087,1732486423713 2024-11-24T22:14:59,185 WARN [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRpcRetryingCaller(177): Call to bd6592c37bb2,40087,1732486423713 for 'test_row_2' in cdb3383a4019da5f5ef0568dfacd542e of TestAcidGuarantees failed, tries = 8, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 8227 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cdb3383a4019da5f5ef0568dfacd542e, server=bd6592c37bb2,40087,1732486423713 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor44.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cdb3383a4019da5f5ef0568dfacd542e, server=bd6592c37bb2,40087,1732486423713 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-24T22:14:59,186 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732486488674.cdb3383a4019da5f5ef0568dfacd542e., hostname=bd6592c37bb2,40087,1732486423713, seqNum=2 , the old value is region=TestAcidGuarantees,,1732486488674.cdb3383a4019da5f5ef0568dfacd542e., hostname=bd6592c37bb2,40087,1732486423713, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cdb3383a4019da5f5ef0568dfacd542e, server=bd6592c37bb2,40087,1732486423713 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T22:14:59,186 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732486488674.cdb3383a4019da5f5ef0568dfacd542e., hostname=bd6592c37bb2,40087,1732486423713, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cdb3383a4019da5f5ef0568dfacd542e, server=bd6592c37bb2,40087,1732486423713 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T22:14:59,186 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732486488674.cdb3383a4019da5f5ef0568dfacd542e., hostname=bd6592c37bb2,40087,1732486423713, seqNum=2 because the exception is null or not the one we care about 2024-11-24T22:14:59,195 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40087 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cdb3383a4019da5f5ef0568dfacd542e, server=bd6592c37bb2,40087,1732486423713 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-24T22:14:59,195 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40087 {}] ipc.CallRunner(138): callId: 18 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40772 deadline: 1732486559191, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cdb3383a4019da5f5ef0568dfacd542e, server=bd6592c37bb2,40087,1732486423713 2024-11-24T22:14:59,196 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40087 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cdb3383a4019da5f5ef0568dfacd542e, server=bd6592c37bb2,40087,1732486423713 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-24T22:14:59,196 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40087 {}] ipc.CallRunner(138): callId: 17 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40842 deadline: 1732486559191, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cdb3383a4019da5f5ef0568dfacd542e, server=bd6592c37bb2,40087,1732486423713 2024-11-24T22:14:59,196 WARN [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRpcRetryingCaller(177): Call to bd6592c37bb2,40087,1732486423713 for 'test_row_2' in cdb3383a4019da5f5ef0568dfacd542e of TestAcidGuarantees failed, tries = 8, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 8240 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cdb3383a4019da5f5ef0568dfacd542e, server=bd6592c37bb2,40087,1732486423713 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor44.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cdb3383a4019da5f5ef0568dfacd542e, server=bd6592c37bb2,40087,1732486423713 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-24T22:14:59,196 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732486488674.cdb3383a4019da5f5ef0568dfacd542e., hostname=bd6592c37bb2,40087,1732486423713, seqNum=2 , the old value is region=TestAcidGuarantees,,1732486488674.cdb3383a4019da5f5ef0568dfacd542e., hostname=bd6592c37bb2,40087,1732486423713, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cdb3383a4019da5f5ef0568dfacd542e, server=bd6592c37bb2,40087,1732486423713 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T22:14:59,196 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732486488674.cdb3383a4019da5f5ef0568dfacd542e., hostname=bd6592c37bb2,40087,1732486423713, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cdb3383a4019da5f5ef0568dfacd542e, server=bd6592c37bb2,40087,1732486423713 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T22:14:59,196 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732486488674.cdb3383a4019da5f5ef0568dfacd542e., hostname=bd6592c37bb2,40087,1732486423713, seqNum=2 because the exception is null or not the one we care about 2024-11-24T22:14:59,196 WARN [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRpcRetryingCaller(177): Call to bd6592c37bb2,40087,1732486423713 for 'test_row_0' in cdb3383a4019da5f5ef0568dfacd542e of TestAcidGuarantees failed, tries = 8, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 8238 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cdb3383a4019da5f5ef0568dfacd542e, server=bd6592c37bb2,40087,1732486423713 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor44.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cdb3383a4019da5f5ef0568dfacd542e, server=bd6592c37bb2,40087,1732486423713 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-24T22:14:59,197 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732486488674.cdb3383a4019da5f5ef0568dfacd542e., hostname=bd6592c37bb2,40087,1732486423713, seqNum=2 , the old value is region=TestAcidGuarantees,,1732486488674.cdb3383a4019da5f5ef0568dfacd542e., hostname=bd6592c37bb2,40087,1732486423713, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cdb3383a4019da5f5ef0568dfacd542e, server=bd6592c37bb2,40087,1732486423713 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T22:14:59,197 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732486488674.cdb3383a4019da5f5ef0568dfacd542e., hostname=bd6592c37bb2,40087,1732486423713, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cdb3383a4019da5f5ef0568dfacd542e, server=bd6592c37bb2,40087,1732486423713 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T22:14:59,197 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732486488674.cdb3383a4019da5f5ef0568dfacd542e., hostname=bd6592c37bb2,40087,1732486423713, seqNum=2 because the exception is null or not the one we care about 2024-11-24T22:14:59,223 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40087 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cdb3383a4019da5f5ef0568dfacd542e, server=bd6592c37bb2,40087,1732486423713 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-24T22:14:59,223 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40087 {}] ipc.CallRunner(138): callId: 182 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40834 deadline: 1732486559220, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cdb3383a4019da5f5ef0568dfacd542e, server=bd6592c37bb2,40087,1732486423713 2024-11-24T22:14:59,224 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732486488674.cdb3383a4019da5f5ef0568dfacd542e., hostname=bd6592c37bb2,40087,1732486423713, seqNum=2 , the old value is region=TestAcidGuarantees,,1732486488674.cdb3383a4019da5f5ef0568dfacd542e., hostname=bd6592c37bb2,40087,1732486423713, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cdb3383a4019da5f5ef0568dfacd542e, server=bd6592c37bb2,40087,1732486423713 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T22:14:59,224 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732486488674.cdb3383a4019da5f5ef0568dfacd542e., hostname=bd6592c37bb2,40087,1732486423713, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cdb3383a4019da5f5ef0568dfacd542e, server=bd6592c37bb2,40087,1732486423713 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T22:14:59,224 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732486488674.cdb3383a4019da5f5ef0568dfacd542e., hostname=bd6592c37bb2,40087,1732486423713, seqNum=2 because the exception is null or not the one we care about 2024-11-24T22:14:59,431 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40087 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cdb3383a4019da5f5ef0568dfacd542e, server=bd6592c37bb2,40087,1732486423713 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-24T22:14:59,431 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40087 {}] ipc.CallRunner(138): callId: 183 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40834 deadline: 1732486559431, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cdb3383a4019da5f5ef0568dfacd542e, server=bd6592c37bb2,40087,1732486423713 2024-11-24T22:14:59,432 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732486488674.cdb3383a4019da5f5ef0568dfacd542e., hostname=bd6592c37bb2,40087,1732486423713, seqNum=2 , the old value is region=TestAcidGuarantees,,1732486488674.cdb3383a4019da5f5ef0568dfacd542e., hostname=bd6592c37bb2,40087,1732486423713, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cdb3383a4019da5f5ef0568dfacd542e, server=bd6592c37bb2,40087,1732486423713 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T22:14:59,432 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732486488674.cdb3383a4019da5f5ef0568dfacd542e., hostname=bd6592c37bb2,40087,1732486423713, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cdb3383a4019da5f5ef0568dfacd542e, server=bd6592c37bb2,40087,1732486423713 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T22:14:59,432 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732486488674.cdb3383a4019da5f5ef0568dfacd542e., hostname=bd6592c37bb2,40087,1732486423713, seqNum=2 because the exception is null or not the one we care about 2024-11-24T22:14:59,546 INFO [RS_FLUSH_OPERATIONS-regionserver/bd6592c37bb2:0-0 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=233 (bloomFilter=true), to=hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/cdb3383a4019da5f5ef0568dfacd542e/.tmp/B/1511cbf759814b93b4166c8ac54f35ca 2024-11-24T22:14:59,555 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bd6592c37bb2:0-0 {event_type=RS_FLUSH_REGIONS, pid=69}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/cdb3383a4019da5f5ef0568dfacd542e/.tmp/C/ab7e06f8486a4a7583de468afccbd8b7 is 50, key is test_row_0/C:col10/1732486497925/Put/seqid=0 2024-11-24T22:14:59,559 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45929 is added to blk_1073742119_1295 (size=12151) 2024-11-24T22:14:59,745 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40087 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cdb3383a4019da5f5ef0568dfacd542e, server=bd6592c37bb2,40087,1732486423713 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-24T22:14:59,746 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40087 {}] ipc.CallRunner(138): callId: 184 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40834 deadline: 1732486559741, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cdb3383a4019da5f5ef0568dfacd542e, server=bd6592c37bb2,40087,1732486423713 2024-11-24T22:14:59,746 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732486488674.cdb3383a4019da5f5ef0568dfacd542e., hostname=bd6592c37bb2,40087,1732486423713, seqNum=2 , the old value is region=TestAcidGuarantees,,1732486488674.cdb3383a4019da5f5ef0568dfacd542e., hostname=bd6592c37bb2,40087,1732486423713, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cdb3383a4019da5f5ef0568dfacd542e, server=bd6592c37bb2,40087,1732486423713 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T22:14:59,746 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732486488674.cdb3383a4019da5f5ef0568dfacd542e., hostname=bd6592c37bb2,40087,1732486423713, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cdb3383a4019da5f5ef0568dfacd542e, server=bd6592c37bb2,40087,1732486423713 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T22:14:59,746 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732486488674.cdb3383a4019da5f5ef0568dfacd542e., hostname=bd6592c37bb2,40087,1732486423713, seqNum=2 because the exception is null or not the one we care about 2024-11-24T22:14:59,960 INFO [RS_FLUSH_OPERATIONS-regionserver/bd6592c37bb2:0-0 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=233 (bloomFilter=true), to=hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/cdb3383a4019da5f5ef0568dfacd542e/.tmp/C/ab7e06f8486a4a7583de468afccbd8b7 2024-11-24T22:14:59,965 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bd6592c37bb2:0-0 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/cdb3383a4019da5f5ef0568dfacd542e/.tmp/A/fd12ade02bb949afabf88523cee4fd62 as hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/cdb3383a4019da5f5ef0568dfacd542e/A/fd12ade02bb949afabf88523cee4fd62 2024-11-24T22:14:59,968 INFO [RS_FLUSH_OPERATIONS-regionserver/bd6592c37bb2:0-0 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/cdb3383a4019da5f5ef0568dfacd542e/A/fd12ade02bb949afabf88523cee4fd62, entries=150, sequenceid=233, filesize=11.9 K 2024-11-24T22:14:59,969 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bd6592c37bb2:0-0 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/cdb3383a4019da5f5ef0568dfacd542e/.tmp/B/1511cbf759814b93b4166c8ac54f35ca as hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/cdb3383a4019da5f5ef0568dfacd542e/B/1511cbf759814b93b4166c8ac54f35ca 2024-11-24T22:14:59,972 INFO [RS_FLUSH_OPERATIONS-regionserver/bd6592c37bb2:0-0 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/cdb3383a4019da5f5ef0568dfacd542e/B/1511cbf759814b93b4166c8ac54f35ca, entries=150, sequenceid=233, filesize=11.9 K 2024-11-24T22:14:59,973 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bd6592c37bb2:0-0 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/cdb3383a4019da5f5ef0568dfacd542e/.tmp/C/ab7e06f8486a4a7583de468afccbd8b7 as hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/cdb3383a4019da5f5ef0568dfacd542e/C/ab7e06f8486a4a7583de468afccbd8b7 2024-11-24T22:14:59,976 INFO [RS_FLUSH_OPERATIONS-regionserver/bd6592c37bb2:0-0 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/cdb3383a4019da5f5ef0568dfacd542e/C/ab7e06f8486a4a7583de468afccbd8b7, entries=150, sequenceid=233, filesize=11.9 K 2024-11-24T22:14:59,977 INFO [RS_FLUSH_OPERATIONS-regionserver/bd6592c37bb2:0-0 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.HRegion(3140): Finished flush of dataSize ~134.18 KB/137400, heapSize ~384.94 KB/394176, currentSize=46.96 KB/48090 for cdb3383a4019da5f5ef0568dfacd542e in 1257ms, sequenceid=233, compaction requested=false 2024-11-24T22:14:59,977 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bd6592c37bb2:0-0 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.HRegion(2603): Flush status journal for cdb3383a4019da5f5ef0568dfacd542e: 2024-11-24T22:14:59,977 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bd6592c37bb2:0-0 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732486488674.cdb3383a4019da5f5ef0568dfacd542e. 2024-11-24T22:14:59,977 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bd6592c37bb2:0-0 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=69 2024-11-24T22:14:59,978 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37701 {}] master.HMaster(4169): Remote procedure done, pid=69 2024-11-24T22:14:59,980 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=69, resume processing ppid=68 2024-11-24T22:14:59,980 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=69, ppid=68, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 2.0240 sec 2024-11-24T22:14:59,982 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=68, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=68, table=TestAcidGuarantees in 2.0300 sec 2024-11-24T22:15:00,101 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37701 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=68 2024-11-24T22:15:00,101 INFO [RPCClient-NioEventLoopGroup-4-6 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-24T22:15:00,102 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37701 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-24T22:15:00,103 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37701 {}] procedure2.ProcedureExecutor(1139): Stored pid=70, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=70, table=TestAcidGuarantees 2024-11-24T22:15:00,104 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37701 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=70 2024-11-24T22:15:00,106 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=70, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=70, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-24T22:15:00,107 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=70, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=70, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-24T22:15:00,107 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=71, ppid=70, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-24T22:15:00,211 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37701 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=70 2024-11-24T22:15:00,253 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40087 {}] regionserver.HRegion(8855): Flush requested on cdb3383a4019da5f5ef0568dfacd542e 2024-11-24T22:15:00,254 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing cdb3383a4019da5f5ef0568dfacd542e 3/3 column families, dataSize=53.67 KB heapSize=160.13 KB 2024-11-24T22:15:00,254 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK cdb3383a4019da5f5ef0568dfacd542e, store=A 2024-11-24T22:15:00,254 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-24T22:15:00,254 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK cdb3383a4019da5f5ef0568dfacd542e, store=B 2024-11-24T22:15:00,254 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-24T22:15:00,254 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK cdb3383a4019da5f5ef0568dfacd542e, store=C 2024-11-24T22:15:00,254 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-24T22:15:00,258 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/cdb3383a4019da5f5ef0568dfacd542e/.tmp/A/22d2943f9286475fbf20ee37ad382e10 is 50, key is test_row_0/A:col10/1732486500252/Put/seqid=0 2024-11-24T22:15:00,259 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=40087 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=71 2024-11-24T22:15:00,259 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bd6592c37bb2:0-1 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732486488674.cdb3383a4019da5f5ef0568dfacd542e. 2024-11-24T22:15:00,259 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bd6592c37bb2:0-1 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732486488674.cdb3383a4019da5f5ef0568dfacd542e. as already flushing 2024-11-24T22:15:00,259 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bd6592c37bb2:0-1 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732486488674.cdb3383a4019da5f5ef0568dfacd542e. 2024-11-24T22:15:00,259 ERROR [RS_FLUSH_OPERATIONS-regionserver/bd6592c37bb2:0-1 {event_type=RS_FLUSH_REGIONS, pid=71}] handler.RSProcedureHandler(58): pid=71 java.io.IOException: Unable to complete flush {ENCODED => cdb3383a4019da5f5ef0568dfacd542e, NAME => 'TestAcidGuarantees,,1732486488674.cdb3383a4019da5f5ef0568dfacd542e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T22:15:00,259 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bd6592c37bb2:0-1 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=71 java.io.IOException: Unable to complete flush {ENCODED => cdb3383a4019da5f5ef0568dfacd542e, NAME => 'TestAcidGuarantees,,1732486488674.cdb3383a4019da5f5ef0568dfacd542e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T22:15:00,260 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37701 {}] master.HMaster(4177): Remote procedure failed, pid=71 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => cdb3383a4019da5f5ef0568dfacd542e, NAME => 'TestAcidGuarantees,,1732486488674.cdb3383a4019da5f5ef0568dfacd542e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => cdb3383a4019da5f5ef0568dfacd542e, NAME => 'TestAcidGuarantees,,1732486488674.cdb3383a4019da5f5ef0568dfacd542e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T22:15:00,278 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45929 is added to blk_1073742120_1296 (size=12147) 2024-11-24T22:15:00,349 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/bd6592c37bb2:0-1 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1732486500302, max timestamp=1732486500345Num uniques 859; 2024-11-24T22:15:00,349 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/bd6592c37bb2:0-1 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-11-24T22:15:00,349 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/bd6592c37bb2:0-2 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1732486500302, max timestamp=1732486500345Num uniques 859; 2024-11-24T22:15:00,349 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/bd6592c37bb2:0-3 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1732486500302, max timestamp=1732486500345Num uniques 859; 2024-11-24T22:15:00,349 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/bd6592c37bb2:0-2 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-11-24T22:15:00,349 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/bd6592c37bb2:0-3 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-11-24T22:15:00,411 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=40087 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=71 2024-11-24T22:15:00,412 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bd6592c37bb2:0-2 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732486488674.cdb3383a4019da5f5ef0568dfacd542e. 2024-11-24T22:15:00,412 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bd6592c37bb2:0-2 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732486488674.cdb3383a4019da5f5ef0568dfacd542e. as already flushing 2024-11-24T22:15:00,412 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bd6592c37bb2:0-2 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732486488674.cdb3383a4019da5f5ef0568dfacd542e. 2024-11-24T22:15:00,412 ERROR [RS_FLUSH_OPERATIONS-regionserver/bd6592c37bb2:0-2 {event_type=RS_FLUSH_REGIONS, pid=71}] handler.RSProcedureHandler(58): pid=71 java.io.IOException: Unable to complete flush {ENCODED => cdb3383a4019da5f5ef0568dfacd542e, NAME => 'TestAcidGuarantees,,1732486488674.cdb3383a4019da5f5ef0568dfacd542e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T22:15:00,412 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bd6592c37bb2:0-2 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=71 java.io.IOException: Unable to complete flush {ENCODED => cdb3383a4019da5f5ef0568dfacd542e, NAME => 'TestAcidGuarantees,,1732486488674.cdb3383a4019da5f5ef0568dfacd542e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T22:15:00,412 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37701 {}] master.HMaster(4177): Remote procedure failed, pid=71 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => cdb3383a4019da5f5ef0568dfacd542e, NAME => 'TestAcidGuarantees,,1732486488674.cdb3383a4019da5f5ef0568dfacd542e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => cdb3383a4019da5f5ef0568dfacd542e, NAME => 'TestAcidGuarantees,,1732486488674.cdb3383a4019da5f5ef0568dfacd542e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T22:15:00,420 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37701 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=70 2024-11-24T22:15:00,438 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40087 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cdb3383a4019da5f5ef0568dfacd542e, server=bd6592c37bb2,40087,1732486423713 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-24T22:15:00,438 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40087 {}] ipc.CallRunner(138): callId: 219 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40834 deadline: 1732486560432, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cdb3383a4019da5f5ef0568dfacd542e, server=bd6592c37bb2,40087,1732486423713 2024-11-24T22:15:00,439 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732486488674.cdb3383a4019da5f5ef0568dfacd542e., hostname=bd6592c37bb2,40087,1732486423713, seqNum=2 , the old value is region=TestAcidGuarantees,,1732486488674.cdb3383a4019da5f5ef0568dfacd542e., hostname=bd6592c37bb2,40087,1732486423713, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cdb3383a4019da5f5ef0568dfacd542e, server=bd6592c37bb2,40087,1732486423713 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T22:15:00,439 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732486488674.cdb3383a4019da5f5ef0568dfacd542e., hostname=bd6592c37bb2,40087,1732486423713, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cdb3383a4019da5f5ef0568dfacd542e, server=bd6592c37bb2,40087,1732486423713 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T22:15:00,439 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732486488674.cdb3383a4019da5f5ef0568dfacd542e., hostname=bd6592c37bb2,40087,1732486423713, seqNum=2 because the exception is null or not the one we care about 2024-11-24T22:15:00,544 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40087 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cdb3383a4019da5f5ef0568dfacd542e, server=bd6592c37bb2,40087,1732486423713 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-24T22:15:00,544 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40087 {}] ipc.CallRunner(138): callId: 220 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40834 deadline: 1732486560540, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cdb3383a4019da5f5ef0568dfacd542e, server=bd6592c37bb2,40087,1732486423713 2024-11-24T22:15:00,544 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732486488674.cdb3383a4019da5f5ef0568dfacd542e., hostname=bd6592c37bb2,40087,1732486423713, seqNum=2 , the old value is region=TestAcidGuarantees,,1732486488674.cdb3383a4019da5f5ef0568dfacd542e., hostname=bd6592c37bb2,40087,1732486423713, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cdb3383a4019da5f5ef0568dfacd542e, server=bd6592c37bb2,40087,1732486423713 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T22:15:00,544 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732486488674.cdb3383a4019da5f5ef0568dfacd542e., hostname=bd6592c37bb2,40087,1732486423713, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cdb3383a4019da5f5ef0568dfacd542e, server=bd6592c37bb2,40087,1732486423713 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T22:15:00,544 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732486488674.cdb3383a4019da5f5ef0568dfacd542e., hostname=bd6592c37bb2,40087,1732486423713, seqNum=2 because the exception is null or not the one we care about 2024-11-24T22:15:00,564 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=40087 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=71 2024-11-24T22:15:00,564 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bd6592c37bb2:0-0 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732486488674.cdb3383a4019da5f5ef0568dfacd542e. 2024-11-24T22:15:00,564 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bd6592c37bb2:0-0 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732486488674.cdb3383a4019da5f5ef0568dfacd542e. as already flushing 2024-11-24T22:15:00,564 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bd6592c37bb2:0-0 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732486488674.cdb3383a4019da5f5ef0568dfacd542e. 2024-11-24T22:15:00,564 ERROR [RS_FLUSH_OPERATIONS-regionserver/bd6592c37bb2:0-0 {event_type=RS_FLUSH_REGIONS, pid=71}] handler.RSProcedureHandler(58): pid=71 java.io.IOException: Unable to complete flush {ENCODED => cdb3383a4019da5f5ef0568dfacd542e, NAME => 'TestAcidGuarantees,,1732486488674.cdb3383a4019da5f5ef0568dfacd542e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T22:15:00,564 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bd6592c37bb2:0-0 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=71 java.io.IOException: Unable to complete flush {ENCODED => cdb3383a4019da5f5ef0568dfacd542e, NAME => 'TestAcidGuarantees,,1732486488674.cdb3383a4019da5f5ef0568dfacd542e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T22:15:00,565 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37701 {}] master.HMaster(4177): Remote procedure failed, pid=71 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => cdb3383a4019da5f5ef0568dfacd542e, NAME => 'TestAcidGuarantees,,1732486488674.cdb3383a4019da5f5ef0568dfacd542e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => cdb3383a4019da5f5ef0568dfacd542e, NAME => 'TestAcidGuarantees,,1732486488674.cdb3383a4019da5f5ef0568dfacd542e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T22:15:00,678 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=246 (bloomFilter=true), to=hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/cdb3383a4019da5f5ef0568dfacd542e/.tmp/A/22d2943f9286475fbf20ee37ad382e10 2024-11-24T22:15:00,686 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/cdb3383a4019da5f5ef0568dfacd542e/.tmp/B/0a697eb0ea8443f0bad97114cc0adf14 is 50, key is test_row_0/B:col10/1732486500252/Put/seqid=0 2024-11-24T22:15:00,689 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45929 is added to blk_1073742121_1297 (size=9757) 2024-11-24T22:15:00,717 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=40087 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=71 2024-11-24T22:15:00,717 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bd6592c37bb2:0-1 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732486488674.cdb3383a4019da5f5ef0568dfacd542e. 2024-11-24T22:15:00,717 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bd6592c37bb2:0-1 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732486488674.cdb3383a4019da5f5ef0568dfacd542e. as already flushing 2024-11-24T22:15:00,717 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bd6592c37bb2:0-1 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732486488674.cdb3383a4019da5f5ef0568dfacd542e. 2024-11-24T22:15:00,717 ERROR [RS_FLUSH_OPERATIONS-regionserver/bd6592c37bb2:0-1 {event_type=RS_FLUSH_REGIONS, pid=71}] handler.RSProcedureHandler(58): pid=71 java.io.IOException: Unable to complete flush {ENCODED => cdb3383a4019da5f5ef0568dfacd542e, NAME => 'TestAcidGuarantees,,1732486488674.cdb3383a4019da5f5ef0568dfacd542e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T22:15:00,717 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bd6592c37bb2:0-1 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=71 java.io.IOException: Unable to complete flush {ENCODED => cdb3383a4019da5f5ef0568dfacd542e, NAME => 'TestAcidGuarantees,,1732486488674.cdb3383a4019da5f5ef0568dfacd542e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T22:15:00,718 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37701 {}] master.HMaster(4177): Remote procedure failed, pid=71 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => cdb3383a4019da5f5ef0568dfacd542e, NAME => 'TestAcidGuarantees,,1732486488674.cdb3383a4019da5f5ef0568dfacd542e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => cdb3383a4019da5f5ef0568dfacd542e, NAME => 'TestAcidGuarantees,,1732486488674.cdb3383a4019da5f5ef0568dfacd542e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T22:15:00,731 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37701 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=70 2024-11-24T22:15:00,755 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40087 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cdb3383a4019da5f5ef0568dfacd542e, server=bd6592c37bb2,40087,1732486423713 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-24T22:15:00,755 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40087 {}] ipc.CallRunner(138): callId: 221 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40834 deadline: 1732486560751, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cdb3383a4019da5f5ef0568dfacd542e, server=bd6592c37bb2,40087,1732486423713 2024-11-24T22:15:00,755 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732486488674.cdb3383a4019da5f5ef0568dfacd542e., hostname=bd6592c37bb2,40087,1732486423713, seqNum=2 , the old value is region=TestAcidGuarantees,,1732486488674.cdb3383a4019da5f5ef0568dfacd542e., hostname=bd6592c37bb2,40087,1732486423713, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cdb3383a4019da5f5ef0568dfacd542e, server=bd6592c37bb2,40087,1732486423713 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T22:15:00,755 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732486488674.cdb3383a4019da5f5ef0568dfacd542e., hostname=bd6592c37bb2,40087,1732486423713, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cdb3383a4019da5f5ef0568dfacd542e, server=bd6592c37bb2,40087,1732486423713 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T22:15:00,755 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732486488674.cdb3383a4019da5f5ef0568dfacd542e., hostname=bd6592c37bb2,40087,1732486423713, seqNum=2 because the exception is null or not the one we care about 2024-11-24T22:15:00,870 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=40087 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=71 2024-11-24T22:15:00,870 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bd6592c37bb2:0-2 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732486488674.cdb3383a4019da5f5ef0568dfacd542e. 2024-11-24T22:15:00,870 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bd6592c37bb2:0-2 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732486488674.cdb3383a4019da5f5ef0568dfacd542e. as already flushing 2024-11-24T22:15:00,870 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bd6592c37bb2:0-2 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732486488674.cdb3383a4019da5f5ef0568dfacd542e. 2024-11-24T22:15:00,870 ERROR [RS_FLUSH_OPERATIONS-regionserver/bd6592c37bb2:0-2 {event_type=RS_FLUSH_REGIONS, pid=71}] handler.RSProcedureHandler(58): pid=71 java.io.IOException: Unable to complete flush {ENCODED => cdb3383a4019da5f5ef0568dfacd542e, NAME => 'TestAcidGuarantees,,1732486488674.cdb3383a4019da5f5ef0568dfacd542e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T22:15:00,870 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bd6592c37bb2:0-2 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=71 java.io.IOException: Unable to complete flush {ENCODED => cdb3383a4019da5f5ef0568dfacd542e, NAME => 'TestAcidGuarantees,,1732486488674.cdb3383a4019da5f5ef0568dfacd542e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T22:15:00,871 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37701 {}] master.HMaster(4177): Remote procedure failed, pid=71 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => cdb3383a4019da5f5ef0568dfacd542e, NAME => 'TestAcidGuarantees,,1732486488674.cdb3383a4019da5f5ef0568dfacd542e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => cdb3383a4019da5f5ef0568dfacd542e, NAME => 'TestAcidGuarantees,,1732486488674.cdb3383a4019da5f5ef0568dfacd542e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T22:15:01,023 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=40087 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=71 2024-11-24T22:15:01,023 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bd6592c37bb2:0-0 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732486488674.cdb3383a4019da5f5ef0568dfacd542e. 2024-11-24T22:15:01,023 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bd6592c37bb2:0-0 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732486488674.cdb3383a4019da5f5ef0568dfacd542e. as already flushing 2024-11-24T22:15:01,023 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bd6592c37bb2:0-0 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732486488674.cdb3383a4019da5f5ef0568dfacd542e. 2024-11-24T22:15:01,023 ERROR [RS_FLUSH_OPERATIONS-regionserver/bd6592c37bb2:0-0 {event_type=RS_FLUSH_REGIONS, pid=71}] handler.RSProcedureHandler(58): pid=71 java.io.IOException: Unable to complete flush {ENCODED => cdb3383a4019da5f5ef0568dfacd542e, NAME => 'TestAcidGuarantees,,1732486488674.cdb3383a4019da5f5ef0568dfacd542e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T22:15:01,024 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bd6592c37bb2:0-0 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=71 java.io.IOException: Unable to complete flush {ENCODED => cdb3383a4019da5f5ef0568dfacd542e, NAME => 'TestAcidGuarantees,,1732486488674.cdb3383a4019da5f5ef0568dfacd542e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T22:15:01,024 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37701 {}] master.HMaster(4177): Remote procedure failed, pid=71 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => cdb3383a4019da5f5ef0568dfacd542e, NAME => 'TestAcidGuarantees,,1732486488674.cdb3383a4019da5f5ef0568dfacd542e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => cdb3383a4019da5f5ef0568dfacd542e, NAME => 'TestAcidGuarantees,,1732486488674.cdb3383a4019da5f5ef0568dfacd542e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T22:15:01,063 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40087 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cdb3383a4019da5f5ef0568dfacd542e, server=bd6592c37bb2,40087,1732486423713 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-24T22:15:01,063 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40087 {}] ipc.CallRunner(138): callId: 222 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40834 deadline: 1732486561061, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cdb3383a4019da5f5ef0568dfacd542e, server=bd6592c37bb2,40087,1732486423713 2024-11-24T22:15:01,064 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732486488674.cdb3383a4019da5f5ef0568dfacd542e., hostname=bd6592c37bb2,40087,1732486423713, seqNum=2 , the old value is region=TestAcidGuarantees,,1732486488674.cdb3383a4019da5f5ef0568dfacd542e., hostname=bd6592c37bb2,40087,1732486423713, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cdb3383a4019da5f5ef0568dfacd542e, server=bd6592c37bb2,40087,1732486423713 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T22:15:01,064 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732486488674.cdb3383a4019da5f5ef0568dfacd542e., hostname=bd6592c37bb2,40087,1732486423713, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cdb3383a4019da5f5ef0568dfacd542e, server=bd6592c37bb2,40087,1732486423713 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T22:15:01,065 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732486488674.cdb3383a4019da5f5ef0568dfacd542e., hostname=bd6592c37bb2,40087,1732486423713, seqNum=2 because the exception is null or not the one we care about 2024-11-24T22:15:01,090 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=246 (bloomFilter=true), to=hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/cdb3383a4019da5f5ef0568dfacd542e/.tmp/B/0a697eb0ea8443f0bad97114cc0adf14 2024-11-24T22:15:01,100 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/cdb3383a4019da5f5ef0568dfacd542e/.tmp/C/541524ca4f294711b5cba8c749f4fcd5 is 50, key is test_row_0/C:col10/1732486500252/Put/seqid=0 2024-11-24T22:15:01,117 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45929 is added to blk_1073742122_1298 (size=9757) 2024-11-24T22:15:01,119 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=246 (bloomFilter=true), to=hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/cdb3383a4019da5f5ef0568dfacd542e/.tmp/C/541524ca4f294711b5cba8c749f4fcd5 2024-11-24T22:15:01,124 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/cdb3383a4019da5f5ef0568dfacd542e/.tmp/A/22d2943f9286475fbf20ee37ad382e10 as hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/cdb3383a4019da5f5ef0568dfacd542e/A/22d2943f9286475fbf20ee37ad382e10 2024-11-24T22:15:01,128 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/cdb3383a4019da5f5ef0568dfacd542e/A/22d2943f9286475fbf20ee37ad382e10, entries=150, sequenceid=246, filesize=11.9 K 2024-11-24T22:15:01,129 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/cdb3383a4019da5f5ef0568dfacd542e/.tmp/B/0a697eb0ea8443f0bad97114cc0adf14 as hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/cdb3383a4019da5f5ef0568dfacd542e/B/0a697eb0ea8443f0bad97114cc0adf14 2024-11-24T22:15:01,133 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/cdb3383a4019da5f5ef0568dfacd542e/B/0a697eb0ea8443f0bad97114cc0adf14, entries=100, sequenceid=246, filesize=9.5 K 2024-11-24T22:15:01,141 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/cdb3383a4019da5f5ef0568dfacd542e/.tmp/C/541524ca4f294711b5cba8c749f4fcd5 as hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/cdb3383a4019da5f5ef0568dfacd542e/C/541524ca4f294711b5cba8c749f4fcd5 2024-11-24T22:15:01,152 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/cdb3383a4019da5f5ef0568dfacd542e/C/541524ca4f294711b5cba8c749f4fcd5, entries=100, sequenceid=246, filesize=9.5 K 2024-11-24T22:15:01,153 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~53.67 KB/54960, heapSize ~160.08 KB/163920, currentSize=127.47 KB/130530 for cdb3383a4019da5f5ef0568dfacd542e in 899ms, sequenceid=246, compaction requested=true 2024-11-24T22:15:01,153 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for cdb3383a4019da5f5ef0568dfacd542e: 2024-11-24T22:15:01,154 DEBUG [RS:0;bd6592c37bb2:40087-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-24T22:15:01,154 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store cdb3383a4019da5f5ef0568dfacd542e:A, priority=-2147483648, current under compaction store size is 1 2024-11-24T22:15:01,155 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-24T22:15:01,155 DEBUG [RS:0;bd6592c37bb2:40087-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-24T22:15:01,155 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store cdb3383a4019da5f5ef0568dfacd542e:B, priority=-2147483648, current under compaction store size is 2 2024-11-24T22:15:01,155 DEBUG [RS:0;bd6592c37bb2:40087-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36961 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-24T22:15:01,155 DEBUG [RS:0;bd6592c37bb2:40087-shortCompactions-0 {}] regionserver.HStore(1541): cdb3383a4019da5f5ef0568dfacd542e/A is initiating minor compaction (all files) 2024-11-24T22:15:01,155 INFO [RS:0;bd6592c37bb2:40087-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of cdb3383a4019da5f5ef0568dfacd542e/A in TestAcidGuarantees,,1732486488674.cdb3383a4019da5f5ef0568dfacd542e. 2024-11-24T22:15:01,155 INFO [RS:0;bd6592c37bb2:40087-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/cdb3383a4019da5f5ef0568dfacd542e/A/79b97ac10a544c658a9ec43f5d94fe7f, hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/cdb3383a4019da5f5ef0568dfacd542e/A/fd12ade02bb949afabf88523cee4fd62, hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/cdb3383a4019da5f5ef0568dfacd542e/A/22d2943f9286475fbf20ee37ad382e10] into tmpdir=hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/cdb3383a4019da5f5ef0568dfacd542e/.tmp, totalSize=36.1 K 2024-11-24T22:15:01,156 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-24T22:15:01,156 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store cdb3383a4019da5f5ef0568dfacd542e:C, priority=-2147483648, current under compaction store size is 3 2024-11-24T22:15:01,156 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-24T22:15:01,156 DEBUG [RS:0;bd6592c37bb2:40087-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 34571 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-24T22:15:01,156 DEBUG [RS:0;bd6592c37bb2:40087-longCompactions-0 {}] regionserver.HStore(1541): cdb3383a4019da5f5ef0568dfacd542e/B is initiating minor compaction (all files) 2024-11-24T22:15:01,157 INFO [RS:0;bd6592c37bb2:40087-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of cdb3383a4019da5f5ef0568dfacd542e/B in TestAcidGuarantees,,1732486488674.cdb3383a4019da5f5ef0568dfacd542e. 2024-11-24T22:15:01,157 INFO [RS:0;bd6592c37bb2:40087-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/cdb3383a4019da5f5ef0568dfacd542e/B/e94ce2756c6743089e95d153fcda9749, hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/cdb3383a4019da5f5ef0568dfacd542e/B/1511cbf759814b93b4166c8ac54f35ca, hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/cdb3383a4019da5f5ef0568dfacd542e/B/0a697eb0ea8443f0bad97114cc0adf14] into tmpdir=hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/cdb3383a4019da5f5ef0568dfacd542e/.tmp, totalSize=33.8 K 2024-11-24T22:15:01,157 DEBUG [RS:0;bd6592c37bb2:40087-shortCompactions-0 {}] compactions.Compactor(225): Compacting 79b97ac10a544c658a9ec43f5d94fe7f, keycount=150, bloomtype=ROW, size=12.4 K, encoding=NONE, compression=NONE, seqNum=195, earliestPutTs=1732486497733 2024-11-24T22:15:01,157 DEBUG [RS:0;bd6592c37bb2:40087-longCompactions-0 {}] compactions.Compactor(225): Compacting e94ce2756c6743089e95d153fcda9749, keycount=150, bloomtype=ROW, size=12.4 K, encoding=NONE, compression=NONE, seqNum=195, earliestPutTs=1732486497733 2024-11-24T22:15:01,157 DEBUG [RS:0;bd6592c37bb2:40087-shortCompactions-0 {}] compactions.Compactor(225): Compacting fd12ade02bb949afabf88523cee4fd62, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=233, earliestPutTs=1732486497885 2024-11-24T22:15:01,158 DEBUG [RS:0;bd6592c37bb2:40087-longCompactions-0 {}] compactions.Compactor(225): Compacting 1511cbf759814b93b4166c8ac54f35ca, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=233, earliestPutTs=1732486497885 2024-11-24T22:15:01,158 DEBUG [RS:0;bd6592c37bb2:40087-shortCompactions-0 {}] compactions.Compactor(225): Compacting 22d2943f9286475fbf20ee37ad382e10, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=246, earliestPutTs=1732486499087 2024-11-24T22:15:01,158 DEBUG [RS:0;bd6592c37bb2:40087-longCompactions-0 {}] compactions.Compactor(225): Compacting 0a697eb0ea8443f0bad97114cc0adf14, keycount=100, bloomtype=ROW, size=9.5 K, encoding=NONE, compression=NONE, seqNum=246, earliestPutTs=1732486499105 2024-11-24T22:15:01,173 INFO [RS:0;bd6592c37bb2:40087-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): cdb3383a4019da5f5ef0568dfacd542e#A#compaction#249 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-24T22:15:01,174 DEBUG [RS:0;bd6592c37bb2:40087-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/cdb3383a4019da5f5ef0568dfacd542e/.tmp/A/e186d6bee8e04f47944534a07e432d52 is 50, key is test_row_0/A:col10/1732486500252/Put/seqid=0 2024-11-24T22:15:01,178 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=40087 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=71 2024-11-24T22:15:01,179 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bd6592c37bb2:0-1 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732486488674.cdb3383a4019da5f5ef0568dfacd542e. 2024-11-24T22:15:01,179 INFO [RS_FLUSH_OPERATIONS-regionserver/bd6592c37bb2:0-1 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.HRegion(2902): Flushing cdb3383a4019da5f5ef0568dfacd542e 3/3 column families, dataSize=127.47 KB heapSize=365.06 KB 2024-11-24T22:15:01,179 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bd6592c37bb2:0-1 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.CompactingMemStore(205): FLUSHING TO DISK cdb3383a4019da5f5ef0568dfacd542e, store=A 2024-11-24T22:15:01,179 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bd6592c37bb2:0-1 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-24T22:15:01,179 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bd6592c37bb2:0-1 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.CompactingMemStore(205): FLUSHING TO DISK cdb3383a4019da5f5ef0568dfacd542e, store=B 2024-11-24T22:15:01,179 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bd6592c37bb2:0-1 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-24T22:15:01,179 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bd6592c37bb2:0-1 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.CompactingMemStore(205): FLUSHING TO DISK cdb3383a4019da5f5ef0568dfacd542e, store=C 2024-11-24T22:15:01,179 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bd6592c37bb2:0-1 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-24T22:15:01,182 INFO [RS:0;bd6592c37bb2:40087-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): cdb3383a4019da5f5ef0568dfacd542e#B#compaction#250 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-24T22:15:01,182 DEBUG [RS:0;bd6592c37bb2:40087-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/cdb3383a4019da5f5ef0568dfacd542e/.tmp/B/5f46e33e87a64df1ac7d18d27391ff55 is 50, key is test_row_0/B:col10/1732486500252/Put/seqid=0 2024-11-24T22:15:01,195 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bd6592c37bb2:0-1 {event_type=RS_FLUSH_REGIONS, pid=71}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/cdb3383a4019da5f5ef0568dfacd542e/.tmp/A/432138b440f142868293ae834c1ed131 is 50, key is test_row_0/A:col10/1732486500423/Put/seqid=0 2024-11-24T22:15:01,195 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45929 is added to blk_1073742123_1299 (size=12765) 2024-11-24T22:15:01,206 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45929 is added to blk_1073742124_1300 (size=12765) 2024-11-24T22:15:01,222 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45929 is added to blk_1073742125_1301 (size=12301) 2024-11-24T22:15:01,223 INFO [RS_FLUSH_OPERATIONS-regionserver/bd6592c37bb2:0-1 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=42.49 KB at sequenceid=282 (bloomFilter=true), to=hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/cdb3383a4019da5f5ef0568dfacd542e/.tmp/A/432138b440f142868293ae834c1ed131 2024-11-24T22:15:01,232 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bd6592c37bb2:0-1 {event_type=RS_FLUSH_REGIONS, pid=71}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/cdb3383a4019da5f5ef0568dfacd542e/.tmp/B/a35577419c784db99f9973e09dcc1e29 is 50, key is test_row_0/B:col10/1732486500423/Put/seqid=0 2024-11-24T22:15:01,240 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37701 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=70 2024-11-24T22:15:01,250 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45929 is added to blk_1073742126_1302 (size=12301) 2024-11-24T22:15:01,576 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40087 {}] regionserver.HRegion(8855): Flush requested on cdb3383a4019da5f5ef0568dfacd542e 2024-11-24T22:15:01,576 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732486488674.cdb3383a4019da5f5ef0568dfacd542e. as already flushing 2024-11-24T22:15:01,603 DEBUG [RS:0;bd6592c37bb2:40087-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/cdb3383a4019da5f5ef0568dfacd542e/.tmp/A/e186d6bee8e04f47944534a07e432d52 as hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/cdb3383a4019da5f5ef0568dfacd542e/A/e186d6bee8e04f47944534a07e432d52 2024-11-24T22:15:01,613 INFO [RS:0;bd6592c37bb2:40087-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in cdb3383a4019da5f5ef0568dfacd542e/A of cdb3383a4019da5f5ef0568dfacd542e into e186d6bee8e04f47944534a07e432d52(size=12.5 K), total size for store is 12.5 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-24T22:15:01,613 DEBUG [RS:0;bd6592c37bb2:40087-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for cdb3383a4019da5f5ef0568dfacd542e: 2024-11-24T22:15:01,613 INFO [RS:0;bd6592c37bb2:40087-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732486488674.cdb3383a4019da5f5ef0568dfacd542e., storeName=cdb3383a4019da5f5ef0568dfacd542e/A, priority=13, startTime=1732486501154; duration=0sec 2024-11-24T22:15:01,613 DEBUG [RS:0;bd6592c37bb2:40087-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-24T22:15:01,613 DEBUG [RS:0;bd6592c37bb2:40087-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: cdb3383a4019da5f5ef0568dfacd542e:A 2024-11-24T22:15:01,613 DEBUG [RS:0;bd6592c37bb2:40087-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-24T22:15:01,614 DEBUG [RS:0;bd6592c37bb2:40087-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/cdb3383a4019da5f5ef0568dfacd542e/.tmp/B/5f46e33e87a64df1ac7d18d27391ff55 as hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/cdb3383a4019da5f5ef0568dfacd542e/B/5f46e33e87a64df1ac7d18d27391ff55 2024-11-24T22:15:01,615 DEBUG [RS:0;bd6592c37bb2:40087-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 34571 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-24T22:15:01,615 DEBUG [RS:0;bd6592c37bb2:40087-shortCompactions-0 {}] regionserver.HStore(1541): cdb3383a4019da5f5ef0568dfacd542e/C is initiating minor compaction (all files) 2024-11-24T22:15:01,615 INFO [RS:0;bd6592c37bb2:40087-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of cdb3383a4019da5f5ef0568dfacd542e/C in TestAcidGuarantees,,1732486488674.cdb3383a4019da5f5ef0568dfacd542e. 2024-11-24T22:15:01,615 INFO [RS:0;bd6592c37bb2:40087-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/cdb3383a4019da5f5ef0568dfacd542e/C/6ad8c8648bc049cdb645eb8af4e16679, hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/cdb3383a4019da5f5ef0568dfacd542e/C/ab7e06f8486a4a7583de468afccbd8b7, hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/cdb3383a4019da5f5ef0568dfacd542e/C/541524ca4f294711b5cba8c749f4fcd5] into tmpdir=hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/cdb3383a4019da5f5ef0568dfacd542e/.tmp, totalSize=33.8 K 2024-11-24T22:15:01,616 DEBUG [RS:0;bd6592c37bb2:40087-shortCompactions-0 {}] compactions.Compactor(225): Compacting 6ad8c8648bc049cdb645eb8af4e16679, keycount=150, bloomtype=ROW, size=12.4 K, encoding=NONE, compression=NONE, seqNum=195, earliestPutTs=1732486497733 2024-11-24T22:15:01,616 DEBUG [RS:0;bd6592c37bb2:40087-shortCompactions-0 {}] compactions.Compactor(225): Compacting ab7e06f8486a4a7583de468afccbd8b7, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=233, earliestPutTs=1732486497885 2024-11-24T22:15:01,617 DEBUG [RS:0;bd6592c37bb2:40087-shortCompactions-0 {}] compactions.Compactor(225): Compacting 541524ca4f294711b5cba8c749f4fcd5, keycount=100, bloomtype=ROW, size=9.5 K, encoding=NONE, compression=NONE, seqNum=246, earliestPutTs=1732486499105 2024-11-24T22:15:01,620 INFO [RS:0;bd6592c37bb2:40087-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in cdb3383a4019da5f5ef0568dfacd542e/B of cdb3383a4019da5f5ef0568dfacd542e into 5f46e33e87a64df1ac7d18d27391ff55(size=12.5 K), total size for store is 12.5 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-24T22:15:01,621 DEBUG [RS:0;bd6592c37bb2:40087-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for cdb3383a4019da5f5ef0568dfacd542e: 2024-11-24T22:15:01,621 INFO [RS:0;bd6592c37bb2:40087-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732486488674.cdb3383a4019da5f5ef0568dfacd542e., storeName=cdb3383a4019da5f5ef0568dfacd542e/B, priority=13, startTime=1732486501155; duration=0sec 2024-11-24T22:15:01,621 DEBUG [RS:0;bd6592c37bb2:40087-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-24T22:15:01,621 DEBUG [RS:0;bd6592c37bb2:40087-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: cdb3383a4019da5f5ef0568dfacd542e:B 2024-11-24T22:15:01,628 INFO [RS:0;bd6592c37bb2:40087-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): cdb3383a4019da5f5ef0568dfacd542e#C#compaction#253 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-24T22:15:01,629 DEBUG [RS:0;bd6592c37bb2:40087-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/cdb3383a4019da5f5ef0568dfacd542e/.tmp/C/4a20a25ec25d4e2ab5355cba35fbd701 is 50, key is test_row_0/C:col10/1732486500252/Put/seqid=0 2024-11-24T22:15:01,643 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40087 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cdb3383a4019da5f5ef0568dfacd542e, server=bd6592c37bb2,40087,1732486423713 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-24T22:15:01,643 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40087 {}] ipc.CallRunner(138): callId: 231 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40834 deadline: 1732486561637, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cdb3383a4019da5f5ef0568dfacd542e, server=bd6592c37bb2,40087,1732486423713 2024-11-24T22:15:01,644 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732486488674.cdb3383a4019da5f5ef0568dfacd542e., hostname=bd6592c37bb2,40087,1732486423713, seqNum=2 , the old value is region=TestAcidGuarantees,,1732486488674.cdb3383a4019da5f5ef0568dfacd542e., hostname=bd6592c37bb2,40087,1732486423713, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cdb3383a4019da5f5ef0568dfacd542e, server=bd6592c37bb2,40087,1732486423713 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T22:15:01,644 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732486488674.cdb3383a4019da5f5ef0568dfacd542e., hostname=bd6592c37bb2,40087,1732486423713, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cdb3383a4019da5f5ef0568dfacd542e, server=bd6592c37bb2,40087,1732486423713 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T22:15:01,644 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732486488674.cdb3383a4019da5f5ef0568dfacd542e., hostname=bd6592c37bb2,40087,1732486423713, seqNum=2 because the exception is null or not the one we care about 2024-11-24T22:15:01,656 INFO [RS_FLUSH_OPERATIONS-regionserver/bd6592c37bb2:0-1 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=42.49 KB at sequenceid=282 (bloomFilter=true), to=hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/cdb3383a4019da5f5ef0568dfacd542e/.tmp/B/a35577419c784db99f9973e09dcc1e29 2024-11-24T22:15:01,658 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45929 is added to blk_1073742127_1303 (size=12765) 2024-11-24T22:15:01,665 DEBUG [RS:0;bd6592c37bb2:40087-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/cdb3383a4019da5f5ef0568dfacd542e/.tmp/C/4a20a25ec25d4e2ab5355cba35fbd701 as hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/cdb3383a4019da5f5ef0568dfacd542e/C/4a20a25ec25d4e2ab5355cba35fbd701 2024-11-24T22:15:01,673 INFO [RS:0;bd6592c37bb2:40087-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in cdb3383a4019da5f5ef0568dfacd542e/C of cdb3383a4019da5f5ef0568dfacd542e into 4a20a25ec25d4e2ab5355cba35fbd701(size=12.5 K), total size for store is 12.5 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-24T22:15:01,673 DEBUG [RS:0;bd6592c37bb2:40087-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for cdb3383a4019da5f5ef0568dfacd542e: 2024-11-24T22:15:01,673 INFO [RS:0;bd6592c37bb2:40087-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732486488674.cdb3383a4019da5f5ef0568dfacd542e., storeName=cdb3383a4019da5f5ef0568dfacd542e/C, priority=13, startTime=1732486501156; duration=0sec 2024-11-24T22:15:01,673 DEBUG [RS:0;bd6592c37bb2:40087-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-24T22:15:01,673 DEBUG [RS:0;bd6592c37bb2:40087-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: cdb3383a4019da5f5ef0568dfacd542e:C 2024-11-24T22:15:01,677 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bd6592c37bb2:0-1 {event_type=RS_FLUSH_REGIONS, pid=71}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/cdb3383a4019da5f5ef0568dfacd542e/.tmp/C/59837082e57443849300f8992554f804 is 50, key is test_row_0/C:col10/1732486500423/Put/seqid=0 2024-11-24T22:15:01,681 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45929 is added to blk_1073742128_1304 (size=12301) 2024-11-24T22:15:01,755 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40087 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cdb3383a4019da5f5ef0568dfacd542e, server=bd6592c37bb2,40087,1732486423713 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-24T22:15:01,755 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40087 {}] ipc.CallRunner(138): callId: 232 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40834 deadline: 1732486561751, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cdb3383a4019da5f5ef0568dfacd542e, server=bd6592c37bb2,40087,1732486423713 2024-11-24T22:15:01,756 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732486488674.cdb3383a4019da5f5ef0568dfacd542e., hostname=bd6592c37bb2,40087,1732486423713, seqNum=2 , the old value is region=TestAcidGuarantees,,1732486488674.cdb3383a4019da5f5ef0568dfacd542e., hostname=bd6592c37bb2,40087,1732486423713, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cdb3383a4019da5f5ef0568dfacd542e, server=bd6592c37bb2,40087,1732486423713 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T22:15:01,756 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732486488674.cdb3383a4019da5f5ef0568dfacd542e., hostname=bd6592c37bb2,40087,1732486423713, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cdb3383a4019da5f5ef0568dfacd542e, server=bd6592c37bb2,40087,1732486423713 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T22:15:01,756 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732486488674.cdb3383a4019da5f5ef0568dfacd542e., hostname=bd6592c37bb2,40087,1732486423713, seqNum=2 because the exception is null or not the one we care about 2024-11-24T22:15:01,963 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40087 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cdb3383a4019da5f5ef0568dfacd542e, server=bd6592c37bb2,40087,1732486423713 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-24T22:15:01,963 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40087 {}] ipc.CallRunner(138): callId: 233 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40834 deadline: 1732486561960, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cdb3383a4019da5f5ef0568dfacd542e, server=bd6592c37bb2,40087,1732486423713 2024-11-24T22:15:01,964 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732486488674.cdb3383a4019da5f5ef0568dfacd542e., hostname=bd6592c37bb2,40087,1732486423713, seqNum=2 , the old value is region=TestAcidGuarantees,,1732486488674.cdb3383a4019da5f5ef0568dfacd542e., hostname=bd6592c37bb2,40087,1732486423713, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cdb3383a4019da5f5ef0568dfacd542e, server=bd6592c37bb2,40087,1732486423713 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T22:15:01,964 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732486488674.cdb3383a4019da5f5ef0568dfacd542e., hostname=bd6592c37bb2,40087,1732486423713, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cdb3383a4019da5f5ef0568dfacd542e, server=bd6592c37bb2,40087,1732486423713 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T22:15:01,964 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732486488674.cdb3383a4019da5f5ef0568dfacd542e., hostname=bd6592c37bb2,40087,1732486423713, seqNum=2 because the exception is null or not the one we care about 2024-11-24T22:15:02,084 INFO [RS_FLUSH_OPERATIONS-regionserver/bd6592c37bb2:0-1 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=42.49 KB at sequenceid=282 (bloomFilter=true), to=hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/cdb3383a4019da5f5ef0568dfacd542e/.tmp/C/59837082e57443849300f8992554f804 2024-11-24T22:15:02,092 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bd6592c37bb2:0-1 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/cdb3383a4019da5f5ef0568dfacd542e/.tmp/A/432138b440f142868293ae834c1ed131 as hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/cdb3383a4019da5f5ef0568dfacd542e/A/432138b440f142868293ae834c1ed131 2024-11-24T22:15:02,099 INFO [RS_FLUSH_OPERATIONS-regionserver/bd6592c37bb2:0-1 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/cdb3383a4019da5f5ef0568dfacd542e/A/432138b440f142868293ae834c1ed131, entries=150, sequenceid=282, filesize=12.0 K 2024-11-24T22:15:02,101 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bd6592c37bb2:0-1 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/cdb3383a4019da5f5ef0568dfacd542e/.tmp/B/a35577419c784db99f9973e09dcc1e29 as hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/cdb3383a4019da5f5ef0568dfacd542e/B/a35577419c784db99f9973e09dcc1e29 2024-11-24T22:15:02,109 INFO [RS_FLUSH_OPERATIONS-regionserver/bd6592c37bb2:0-1 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/cdb3383a4019da5f5ef0568dfacd542e/B/a35577419c784db99f9973e09dcc1e29, entries=150, sequenceid=282, filesize=12.0 K 2024-11-24T22:15:02,111 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bd6592c37bb2:0-1 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/cdb3383a4019da5f5ef0568dfacd542e/.tmp/C/59837082e57443849300f8992554f804 as hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/cdb3383a4019da5f5ef0568dfacd542e/C/59837082e57443849300f8992554f804 2024-11-24T22:15:02,119 INFO [RS_FLUSH_OPERATIONS-regionserver/bd6592c37bb2:0-1 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/cdb3383a4019da5f5ef0568dfacd542e/C/59837082e57443849300f8992554f804, entries=150, sequenceid=282, filesize=12.0 K 2024-11-24T22:15:02,121 INFO [RS_FLUSH_OPERATIONS-regionserver/bd6592c37bb2:0-1 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.HRegion(3140): Finished flush of dataSize ~127.47 KB/130530, heapSize ~365.02 KB/373776, currentSize=53.67 KB/54960 for cdb3383a4019da5f5ef0568dfacd542e in 942ms, sequenceid=282, compaction requested=false 2024-11-24T22:15:02,121 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bd6592c37bb2:0-1 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.HRegion(2603): Flush status journal for cdb3383a4019da5f5ef0568dfacd542e: 2024-11-24T22:15:02,121 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bd6592c37bb2:0-1 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732486488674.cdb3383a4019da5f5ef0568dfacd542e. 2024-11-24T22:15:02,121 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bd6592c37bb2:0-1 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=71 2024-11-24T22:15:02,121 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37701 {}] master.HMaster(4169): Remote procedure done, pid=71 2024-11-24T22:15:02,125 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=71, resume processing ppid=70 2024-11-24T22:15:02,125 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=71, ppid=70, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 2.0160 sec 2024-11-24T22:15:02,127 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=70, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=70, table=TestAcidGuarantees in 2.0230 sec 2024-11-24T22:15:02,250 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37701 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=70 2024-11-24T22:15:02,251 INFO [RPCClient-NioEventLoopGroup-4-6 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-24T22:15:02,251 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37701 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-24T22:15:02,252 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37701 {}] procedure2.ProcedureExecutor(1139): Stored pid=72, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=72, table=TestAcidGuarantees 2024-11-24T22:15:02,253 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37701 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=72 2024-11-24T22:15:02,254 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=72, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=72, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-24T22:15:02,255 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=72, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=72, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-24T22:15:02,255 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=73, ppid=72, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-24T22:15:02,273 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40087 {}] regionserver.HRegion(8855): Flush requested on cdb3383a4019da5f5ef0568dfacd542e 2024-11-24T22:15:02,273 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing cdb3383a4019da5f5ef0568dfacd542e 3/3 column families, dataSize=60.38 KB heapSize=180.05 KB 2024-11-24T22:15:02,274 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK cdb3383a4019da5f5ef0568dfacd542e, store=A 2024-11-24T22:15:02,274 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-24T22:15:02,274 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK cdb3383a4019da5f5ef0568dfacd542e, store=B 2024-11-24T22:15:02,274 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-24T22:15:02,274 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK cdb3383a4019da5f5ef0568dfacd542e, store=C 2024-11-24T22:15:02,274 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-24T22:15:02,301 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/cdb3383a4019da5f5ef0568dfacd542e/.tmp/A/ff0660ba25d7415a932d59d3791e2c25 is 50, key is test_row_0/A:col10/1732486501635/Put/seqid=0 2024-11-24T22:15:02,306 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45929 is added to blk_1073742129_1305 (size=14741) 2024-11-24T22:15:02,361 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37701 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=72 2024-11-24T22:15:02,392 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40087 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cdb3383a4019da5f5ef0568dfacd542e, server=bd6592c37bb2,40087,1732486423713 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-24T22:15:02,392 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40087 {}] ipc.CallRunner(138): callId: 252 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40834 deadline: 1732486562388, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cdb3383a4019da5f5ef0568dfacd542e, server=bd6592c37bb2,40087,1732486423713 2024-11-24T22:15:02,392 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732486488674.cdb3383a4019da5f5ef0568dfacd542e., hostname=bd6592c37bb2,40087,1732486423713, seqNum=2 , the old value is region=TestAcidGuarantees,,1732486488674.cdb3383a4019da5f5ef0568dfacd542e., hostname=bd6592c37bb2,40087,1732486423713, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cdb3383a4019da5f5ef0568dfacd542e, server=bd6592c37bb2,40087,1732486423713 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T22:15:02,392 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732486488674.cdb3383a4019da5f5ef0568dfacd542e., hostname=bd6592c37bb2,40087,1732486423713, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cdb3383a4019da5f5ef0568dfacd542e, server=bd6592c37bb2,40087,1732486423713 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T22:15:02,392 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732486488674.cdb3383a4019da5f5ef0568dfacd542e., hostname=bd6592c37bb2,40087,1732486423713, seqNum=2 because the exception is null or not the one we care about 2024-11-24T22:15:02,408 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=40087 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=73 2024-11-24T22:15:02,408 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bd6592c37bb2:0-2 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732486488674.cdb3383a4019da5f5ef0568dfacd542e. 2024-11-24T22:15:02,408 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bd6592c37bb2:0-2 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732486488674.cdb3383a4019da5f5ef0568dfacd542e. as already flushing 2024-11-24T22:15:02,408 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bd6592c37bb2:0-2 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732486488674.cdb3383a4019da5f5ef0568dfacd542e. 2024-11-24T22:15:02,408 ERROR [RS_FLUSH_OPERATIONS-regionserver/bd6592c37bb2:0-2 {event_type=RS_FLUSH_REGIONS, pid=73}] handler.RSProcedureHandler(58): pid=73 java.io.IOException: Unable to complete flush {ENCODED => cdb3383a4019da5f5ef0568dfacd542e, NAME => 'TestAcidGuarantees,,1732486488674.cdb3383a4019da5f5ef0568dfacd542e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T22:15:02,408 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bd6592c37bb2:0-2 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=73 java.io.IOException: Unable to complete flush {ENCODED => cdb3383a4019da5f5ef0568dfacd542e, NAME => 'TestAcidGuarantees,,1732486488674.cdb3383a4019da5f5ef0568dfacd542e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T22:15:02,409 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37701 {}] master.HMaster(4177): Remote procedure failed, pid=73 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => cdb3383a4019da5f5ef0568dfacd542e, NAME => 'TestAcidGuarantees,,1732486488674.cdb3383a4019da5f5ef0568dfacd542e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => cdb3383a4019da5f5ef0568dfacd542e, NAME => 'TestAcidGuarantees,,1732486488674.cdb3383a4019da5f5ef0568dfacd542e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T22:15:02,504 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40087 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cdb3383a4019da5f5ef0568dfacd542e, server=bd6592c37bb2,40087,1732486423713 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-24T22:15:02,504 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40087 {}] ipc.CallRunner(138): callId: 253 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40834 deadline: 1732486562501, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cdb3383a4019da5f5ef0568dfacd542e, server=bd6592c37bb2,40087,1732486423713 2024-11-24T22:15:02,505 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732486488674.cdb3383a4019da5f5ef0568dfacd542e., hostname=bd6592c37bb2,40087,1732486423713, seqNum=2 , the old value is region=TestAcidGuarantees,,1732486488674.cdb3383a4019da5f5ef0568dfacd542e., hostname=bd6592c37bb2,40087,1732486423713, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cdb3383a4019da5f5ef0568dfacd542e, server=bd6592c37bb2,40087,1732486423713 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T22:15:02,505 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732486488674.cdb3383a4019da5f5ef0568dfacd542e., hostname=bd6592c37bb2,40087,1732486423713, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cdb3383a4019da5f5ef0568dfacd542e, server=bd6592c37bb2,40087,1732486423713 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T22:15:02,505 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732486488674.cdb3383a4019da5f5ef0568dfacd542e., hostname=bd6592c37bb2,40087,1732486423713, seqNum=2 because the exception is null or not the one we care about 2024-11-24T22:15:02,561 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=40087 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=73 2024-11-24T22:15:02,562 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bd6592c37bb2:0-0 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732486488674.cdb3383a4019da5f5ef0568dfacd542e. 2024-11-24T22:15:02,562 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bd6592c37bb2:0-0 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732486488674.cdb3383a4019da5f5ef0568dfacd542e. as already flushing 2024-11-24T22:15:02,562 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bd6592c37bb2:0-0 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732486488674.cdb3383a4019da5f5ef0568dfacd542e. 2024-11-24T22:15:02,562 ERROR [RS_FLUSH_OPERATIONS-regionserver/bd6592c37bb2:0-0 {event_type=RS_FLUSH_REGIONS, pid=73}] handler.RSProcedureHandler(58): pid=73 java.io.IOException: Unable to complete flush {ENCODED => cdb3383a4019da5f5ef0568dfacd542e, NAME => 'TestAcidGuarantees,,1732486488674.cdb3383a4019da5f5ef0568dfacd542e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T22:15:02,562 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bd6592c37bb2:0-0 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=73 java.io.IOException: Unable to complete flush {ENCODED => cdb3383a4019da5f5ef0568dfacd542e, NAME => 'TestAcidGuarantees,,1732486488674.cdb3383a4019da5f5ef0568dfacd542e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T22:15:02,563 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37701 {}] master.HMaster(4177): Remote procedure failed, pid=73 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => cdb3383a4019da5f5ef0568dfacd542e, NAME => 'TestAcidGuarantees,,1732486488674.cdb3383a4019da5f5ef0568dfacd542e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => cdb3383a4019da5f5ef0568dfacd542e, NAME => 'TestAcidGuarantees,,1732486488674.cdb3383a4019da5f5ef0568dfacd542e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T22:15:02,571 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37701 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=72 2024-11-24T22:15:02,707 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=297 (bloomFilter=true), to=hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/cdb3383a4019da5f5ef0568dfacd542e/.tmp/A/ff0660ba25d7415a932d59d3791e2c25 2024-11-24T22:15:02,713 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40087 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cdb3383a4019da5f5ef0568dfacd542e, server=bd6592c37bb2,40087,1732486423713 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-24T22:15:02,714 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40087 {}] ipc.CallRunner(138): callId: 254 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40834 deadline: 1732486562711, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cdb3383a4019da5f5ef0568dfacd542e, server=bd6592c37bb2,40087,1732486423713 2024-11-24T22:15:02,714 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732486488674.cdb3383a4019da5f5ef0568dfacd542e., hostname=bd6592c37bb2,40087,1732486423713, seqNum=2 , the old value is region=TestAcidGuarantees,,1732486488674.cdb3383a4019da5f5ef0568dfacd542e., hostname=bd6592c37bb2,40087,1732486423713, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cdb3383a4019da5f5ef0568dfacd542e, server=bd6592c37bb2,40087,1732486423713 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T22:15:02,714 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732486488674.cdb3383a4019da5f5ef0568dfacd542e., hostname=bd6592c37bb2,40087,1732486423713, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cdb3383a4019da5f5ef0568dfacd542e, server=bd6592c37bb2,40087,1732486423713 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T22:15:02,714 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732486488674.cdb3383a4019da5f5ef0568dfacd542e., hostname=bd6592c37bb2,40087,1732486423713, seqNum=2 because the exception is null or not the one we care about 2024-11-24T22:15:02,714 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/cdb3383a4019da5f5ef0568dfacd542e/.tmp/B/043730d19ec944f5b2bb93ed6b5bfdbd is 50, key is test_row_0/B:col10/1732486501635/Put/seqid=0 2024-11-24T22:15:02,715 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=40087 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=73 2024-11-24T22:15:02,715 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bd6592c37bb2:0-1 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732486488674.cdb3383a4019da5f5ef0568dfacd542e. 2024-11-24T22:15:02,715 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bd6592c37bb2:0-1 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732486488674.cdb3383a4019da5f5ef0568dfacd542e. as already flushing 2024-11-24T22:15:02,715 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bd6592c37bb2:0-1 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732486488674.cdb3383a4019da5f5ef0568dfacd542e. 2024-11-24T22:15:02,715 ERROR [RS_FLUSH_OPERATIONS-regionserver/bd6592c37bb2:0-1 {event_type=RS_FLUSH_REGIONS, pid=73}] handler.RSProcedureHandler(58): pid=73 java.io.IOException: Unable to complete flush {ENCODED => cdb3383a4019da5f5ef0568dfacd542e, NAME => 'TestAcidGuarantees,,1732486488674.cdb3383a4019da5f5ef0568dfacd542e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T22:15:02,715 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bd6592c37bb2:0-1 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=73 java.io.IOException: Unable to complete flush {ENCODED => cdb3383a4019da5f5ef0568dfacd542e, NAME => 'TestAcidGuarantees,,1732486488674.cdb3383a4019da5f5ef0568dfacd542e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T22:15:02,716 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37701 {}] master.HMaster(4177): Remote procedure failed, pid=73 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => cdb3383a4019da5f5ef0568dfacd542e, NAME => 'TestAcidGuarantees,,1732486488674.cdb3383a4019da5f5ef0568dfacd542e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => cdb3383a4019da5f5ef0568dfacd542e, NAME => 'TestAcidGuarantees,,1732486488674.cdb3383a4019da5f5ef0568dfacd542e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T22:15:02,719 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45929 is added to blk_1073742130_1306 (size=12301) 2024-11-24T22:15:02,867 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=40087 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=73 2024-11-24T22:15:02,868 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bd6592c37bb2:0-2 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732486488674.cdb3383a4019da5f5ef0568dfacd542e. 2024-11-24T22:15:02,868 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bd6592c37bb2:0-2 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732486488674.cdb3383a4019da5f5ef0568dfacd542e. as already flushing 2024-11-24T22:15:02,868 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bd6592c37bb2:0-2 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732486488674.cdb3383a4019da5f5ef0568dfacd542e. 2024-11-24T22:15:02,868 ERROR [RS_FLUSH_OPERATIONS-regionserver/bd6592c37bb2:0-2 {event_type=RS_FLUSH_REGIONS, pid=73}] handler.RSProcedureHandler(58): pid=73 java.io.IOException: Unable to complete flush {ENCODED => cdb3383a4019da5f5ef0568dfacd542e, NAME => 'TestAcidGuarantees,,1732486488674.cdb3383a4019da5f5ef0568dfacd542e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T22:15:02,868 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bd6592c37bb2:0-2 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=73 java.io.IOException: Unable to complete flush {ENCODED => cdb3383a4019da5f5ef0568dfacd542e, NAME => 'TestAcidGuarantees,,1732486488674.cdb3383a4019da5f5ef0568dfacd542e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T22:15:02,869 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37701 {}] master.HMaster(4177): Remote procedure failed, pid=73 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => cdb3383a4019da5f5ef0568dfacd542e, NAME => 'TestAcidGuarantees,,1732486488674.cdb3383a4019da5f5ef0568dfacd542e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => cdb3383a4019da5f5ef0568dfacd542e, NAME => 'TestAcidGuarantees,,1732486488674.cdb3383a4019da5f5ef0568dfacd542e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T22:15:02,881 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37701 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=72 2024-11-24T22:15:03,021 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=40087 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=73 2024-11-24T22:15:03,021 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bd6592c37bb2:0-0 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732486488674.cdb3383a4019da5f5ef0568dfacd542e. 2024-11-24T22:15:03,021 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bd6592c37bb2:0-0 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732486488674.cdb3383a4019da5f5ef0568dfacd542e. as already flushing 2024-11-24T22:15:03,021 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bd6592c37bb2:0-0 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732486488674.cdb3383a4019da5f5ef0568dfacd542e. 2024-11-24T22:15:03,021 ERROR [RS_FLUSH_OPERATIONS-regionserver/bd6592c37bb2:0-0 {event_type=RS_FLUSH_REGIONS, pid=73}] handler.RSProcedureHandler(58): pid=73 java.io.IOException: Unable to complete flush {ENCODED => cdb3383a4019da5f5ef0568dfacd542e, NAME => 'TestAcidGuarantees,,1732486488674.cdb3383a4019da5f5ef0568dfacd542e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T22:15:03,021 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bd6592c37bb2:0-0 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=73 java.io.IOException: Unable to complete flush {ENCODED => cdb3383a4019da5f5ef0568dfacd542e, NAME => 'TestAcidGuarantees,,1732486488674.cdb3383a4019da5f5ef0568dfacd542e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T22:15:03,022 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37701 {}] master.HMaster(4177): Remote procedure failed, pid=73 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => cdb3383a4019da5f5ef0568dfacd542e, NAME => 'TestAcidGuarantees,,1732486488674.cdb3383a4019da5f5ef0568dfacd542e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => cdb3383a4019da5f5ef0568dfacd542e, NAME => 'TestAcidGuarantees,,1732486488674.cdb3383a4019da5f5ef0568dfacd542e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T22:15:03,023 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40087 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cdb3383a4019da5f5ef0568dfacd542e, server=bd6592c37bb2,40087,1732486423713 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-24T22:15:03,024 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40087 {}] ipc.CallRunner(138): callId: 255 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40834 deadline: 1732486563020, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cdb3383a4019da5f5ef0568dfacd542e, server=bd6592c37bb2,40087,1732486423713 2024-11-24T22:15:03,024 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732486488674.cdb3383a4019da5f5ef0568dfacd542e., hostname=bd6592c37bb2,40087,1732486423713, seqNum=2 , the old value is region=TestAcidGuarantees,,1732486488674.cdb3383a4019da5f5ef0568dfacd542e., hostname=bd6592c37bb2,40087,1732486423713, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cdb3383a4019da5f5ef0568dfacd542e, server=bd6592c37bb2,40087,1732486423713 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T22:15:03,024 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732486488674.cdb3383a4019da5f5ef0568dfacd542e., hostname=bd6592c37bb2,40087,1732486423713, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cdb3383a4019da5f5ef0568dfacd542e, server=bd6592c37bb2,40087,1732486423713 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T22:15:03,024 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732486488674.cdb3383a4019da5f5ef0568dfacd542e., hostname=bd6592c37bb2,40087,1732486423713, seqNum=2 because the exception is null or not the one we care about 2024-11-24T22:15:03,119 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=297 (bloomFilter=true), to=hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/cdb3383a4019da5f5ef0568dfacd542e/.tmp/B/043730d19ec944f5b2bb93ed6b5bfdbd 2024-11-24T22:15:03,127 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/cdb3383a4019da5f5ef0568dfacd542e/.tmp/C/58c7394de6c14021aae54876cf65fa09 is 50, key is test_row_0/C:col10/1732486501635/Put/seqid=0 2024-11-24T22:15:03,130 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45929 is added to blk_1073742131_1307 (size=12301) 2024-11-24T22:15:03,174 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=40087 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=73 2024-11-24T22:15:03,174 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bd6592c37bb2:0-1 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732486488674.cdb3383a4019da5f5ef0568dfacd542e. 2024-11-24T22:15:03,174 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bd6592c37bb2:0-1 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732486488674.cdb3383a4019da5f5ef0568dfacd542e. as already flushing 2024-11-24T22:15:03,174 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bd6592c37bb2:0-1 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732486488674.cdb3383a4019da5f5ef0568dfacd542e. 2024-11-24T22:15:03,174 ERROR [RS_FLUSH_OPERATIONS-regionserver/bd6592c37bb2:0-1 {event_type=RS_FLUSH_REGIONS, pid=73}] handler.RSProcedureHandler(58): pid=73 java.io.IOException: Unable to complete flush {ENCODED => cdb3383a4019da5f5ef0568dfacd542e, NAME => 'TestAcidGuarantees,,1732486488674.cdb3383a4019da5f5ef0568dfacd542e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T22:15:03,174 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bd6592c37bb2:0-1 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=73 java.io.IOException: Unable to complete flush {ENCODED => cdb3383a4019da5f5ef0568dfacd542e, NAME => 'TestAcidGuarantees,,1732486488674.cdb3383a4019da5f5ef0568dfacd542e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T22:15:03,175 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37701 {}] master.HMaster(4177): Remote procedure failed, pid=73 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => cdb3383a4019da5f5ef0568dfacd542e, NAME => 'TestAcidGuarantees,,1732486488674.cdb3383a4019da5f5ef0568dfacd542e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => cdb3383a4019da5f5ef0568dfacd542e, NAME => 'TestAcidGuarantees,,1732486488674.cdb3383a4019da5f5ef0568dfacd542e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T22:15:03,327 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=40087 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=73 2024-11-24T22:15:03,327 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bd6592c37bb2:0-2 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732486488674.cdb3383a4019da5f5ef0568dfacd542e. 2024-11-24T22:15:03,327 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bd6592c37bb2:0-2 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732486488674.cdb3383a4019da5f5ef0568dfacd542e. as already flushing 2024-11-24T22:15:03,327 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bd6592c37bb2:0-2 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732486488674.cdb3383a4019da5f5ef0568dfacd542e. 2024-11-24T22:15:03,327 ERROR [RS_FLUSH_OPERATIONS-regionserver/bd6592c37bb2:0-2 {event_type=RS_FLUSH_REGIONS, pid=73}] handler.RSProcedureHandler(58): pid=73 java.io.IOException: Unable to complete flush {ENCODED => cdb3383a4019da5f5ef0568dfacd542e, NAME => 'TestAcidGuarantees,,1732486488674.cdb3383a4019da5f5ef0568dfacd542e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T22:15:03,327 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bd6592c37bb2:0-2 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=73 java.io.IOException: Unable to complete flush {ENCODED => cdb3383a4019da5f5ef0568dfacd542e, NAME => 'TestAcidGuarantees,,1732486488674.cdb3383a4019da5f5ef0568dfacd542e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T22:15:03,328 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37701 {}] master.HMaster(4177): Remote procedure failed, pid=73 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => cdb3383a4019da5f5ef0568dfacd542e, NAME => 'TestAcidGuarantees,,1732486488674.cdb3383a4019da5f5ef0568dfacd542e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => cdb3383a4019da5f5ef0568dfacd542e, NAME => 'TestAcidGuarantees,,1732486488674.cdb3383a4019da5f5ef0568dfacd542e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T22:15:03,391 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37701 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=72 2024-11-24T22:15:03,480 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=40087 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=73 2024-11-24T22:15:03,480 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bd6592c37bb2:0-0 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732486488674.cdb3383a4019da5f5ef0568dfacd542e. 2024-11-24T22:15:03,480 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bd6592c37bb2:0-0 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732486488674.cdb3383a4019da5f5ef0568dfacd542e. as already flushing 2024-11-24T22:15:03,480 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bd6592c37bb2:0-0 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732486488674.cdb3383a4019da5f5ef0568dfacd542e. 2024-11-24T22:15:03,480 ERROR [RS_FLUSH_OPERATIONS-regionserver/bd6592c37bb2:0-0 {event_type=RS_FLUSH_REGIONS, pid=73}] handler.RSProcedureHandler(58): pid=73 java.io.IOException: Unable to complete flush {ENCODED => cdb3383a4019da5f5ef0568dfacd542e, NAME => 'TestAcidGuarantees,,1732486488674.cdb3383a4019da5f5ef0568dfacd542e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T22:15:03,480 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bd6592c37bb2:0-0 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=73 java.io.IOException: Unable to complete flush {ENCODED => cdb3383a4019da5f5ef0568dfacd542e, NAME => 'TestAcidGuarantees,,1732486488674.cdb3383a4019da5f5ef0568dfacd542e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T22:15:03,481 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37701 {}] master.HMaster(4177): Remote procedure failed, pid=73 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => cdb3383a4019da5f5ef0568dfacd542e, NAME => 'TestAcidGuarantees,,1732486488674.cdb3383a4019da5f5ef0568dfacd542e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => cdb3383a4019da5f5ef0568dfacd542e, NAME => 'TestAcidGuarantees,,1732486488674.cdb3383a4019da5f5ef0568dfacd542e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T22:15:03,531 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=297 (bloomFilter=true), to=hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/cdb3383a4019da5f5ef0568dfacd542e/.tmp/C/58c7394de6c14021aae54876cf65fa09 2024-11-24T22:15:03,535 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40087 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cdb3383a4019da5f5ef0568dfacd542e, server=bd6592c37bb2,40087,1732486423713 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-24T22:15:03,535 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40087 {}] ipc.CallRunner(138): callId: 256 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40834 deadline: 1732486563531, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cdb3383a4019da5f5ef0568dfacd542e, server=bd6592c37bb2,40087,1732486423713 2024-11-24T22:15:03,535 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732486488674.cdb3383a4019da5f5ef0568dfacd542e., hostname=bd6592c37bb2,40087,1732486423713, seqNum=2 , the old value is region=TestAcidGuarantees,,1732486488674.cdb3383a4019da5f5ef0568dfacd542e., hostname=bd6592c37bb2,40087,1732486423713, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cdb3383a4019da5f5ef0568dfacd542e, server=bd6592c37bb2,40087,1732486423713 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T22:15:03,535 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732486488674.cdb3383a4019da5f5ef0568dfacd542e., hostname=bd6592c37bb2,40087,1732486423713, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cdb3383a4019da5f5ef0568dfacd542e, server=bd6592c37bb2,40087,1732486423713 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T22:15:03,536 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732486488674.cdb3383a4019da5f5ef0568dfacd542e., hostname=bd6592c37bb2,40087,1732486423713, seqNum=2 because the exception is null or not the one we care about 2024-11-24T22:15:03,536 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/cdb3383a4019da5f5ef0568dfacd542e/.tmp/A/ff0660ba25d7415a932d59d3791e2c25 as hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/cdb3383a4019da5f5ef0568dfacd542e/A/ff0660ba25d7415a932d59d3791e2c25 2024-11-24T22:15:03,540 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/cdb3383a4019da5f5ef0568dfacd542e/A/ff0660ba25d7415a932d59d3791e2c25, entries=200, sequenceid=297, filesize=14.4 K 2024-11-24T22:15:03,541 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/cdb3383a4019da5f5ef0568dfacd542e/.tmp/B/043730d19ec944f5b2bb93ed6b5bfdbd as hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/cdb3383a4019da5f5ef0568dfacd542e/B/043730d19ec944f5b2bb93ed6b5bfdbd 2024-11-24T22:15:03,544 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/cdb3383a4019da5f5ef0568dfacd542e/B/043730d19ec944f5b2bb93ed6b5bfdbd, entries=150, sequenceid=297, filesize=12.0 K 2024-11-24T22:15:03,545 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/cdb3383a4019da5f5ef0568dfacd542e/.tmp/C/58c7394de6c14021aae54876cf65fa09 as hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/cdb3383a4019da5f5ef0568dfacd542e/C/58c7394de6c14021aae54876cf65fa09 2024-11-24T22:15:03,549 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/cdb3383a4019da5f5ef0568dfacd542e/C/58c7394de6c14021aae54876cf65fa09, entries=150, sequenceid=297, filesize=12.0 K 2024-11-24T22:15:03,549 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~60.38 KB/61830, heapSize ~180 KB/184320, currentSize=114.05 KB/116790 for cdb3383a4019da5f5ef0568dfacd542e in 1276ms, sequenceid=297, compaction requested=true 2024-11-24T22:15:03,550 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for cdb3383a4019da5f5ef0568dfacd542e: 2024-11-24T22:15:03,550 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store cdb3383a4019da5f5ef0568dfacd542e:A, priority=-2147483648, current under compaction store size is 1 2024-11-24T22:15:03,550 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-24T22:15:03,550 DEBUG [RS:0;bd6592c37bb2:40087-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-24T22:15:03,550 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store cdb3383a4019da5f5ef0568dfacd542e:B, priority=-2147483648, current under compaction store size is 2 2024-11-24T22:15:03,550 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-24T22:15:03,550 DEBUG [RS:0;bd6592c37bb2:40087-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-24T22:15:03,550 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store cdb3383a4019da5f5ef0568dfacd542e:C, priority=-2147483648, current under compaction store size is 3 2024-11-24T22:15:03,550 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-24T22:15:03,551 DEBUG [RS:0;bd6592c37bb2:40087-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 39807 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-24T22:15:03,551 DEBUG [RS:0;bd6592c37bb2:40087-shortCompactions-0 {}] regionserver.HStore(1541): cdb3383a4019da5f5ef0568dfacd542e/A is initiating minor compaction (all files) 2024-11-24T22:15:03,551 INFO [RS:0;bd6592c37bb2:40087-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of cdb3383a4019da5f5ef0568dfacd542e/A in TestAcidGuarantees,,1732486488674.cdb3383a4019da5f5ef0568dfacd542e. 2024-11-24T22:15:03,551 INFO [RS:0;bd6592c37bb2:40087-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/cdb3383a4019da5f5ef0568dfacd542e/A/e186d6bee8e04f47944534a07e432d52, hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/cdb3383a4019da5f5ef0568dfacd542e/A/432138b440f142868293ae834c1ed131, hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/cdb3383a4019da5f5ef0568dfacd542e/A/ff0660ba25d7415a932d59d3791e2c25] into tmpdir=hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/cdb3383a4019da5f5ef0568dfacd542e/.tmp, totalSize=38.9 K 2024-11-24T22:15:03,551 DEBUG [RS:0;bd6592c37bb2:40087-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37367 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-24T22:15:03,551 DEBUG [RS:0;bd6592c37bb2:40087-longCompactions-0 {}] regionserver.HStore(1541): cdb3383a4019da5f5ef0568dfacd542e/B is initiating minor compaction (all files) 2024-11-24T22:15:03,551 DEBUG [RS:0;bd6592c37bb2:40087-shortCompactions-0 {}] compactions.Compactor(225): Compacting e186d6bee8e04f47944534a07e432d52, keycount=150, bloomtype=ROW, size=12.5 K, encoding=NONE, compression=NONE, seqNum=246, earliestPutTs=1732486497885 2024-11-24T22:15:03,551 INFO [RS:0;bd6592c37bb2:40087-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of cdb3383a4019da5f5ef0568dfacd542e/B in TestAcidGuarantees,,1732486488674.cdb3383a4019da5f5ef0568dfacd542e. 2024-11-24T22:15:03,551 INFO [RS:0;bd6592c37bb2:40087-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/cdb3383a4019da5f5ef0568dfacd542e/B/5f46e33e87a64df1ac7d18d27391ff55, hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/cdb3383a4019da5f5ef0568dfacd542e/B/a35577419c784db99f9973e09dcc1e29, hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/cdb3383a4019da5f5ef0568dfacd542e/B/043730d19ec944f5b2bb93ed6b5bfdbd] into tmpdir=hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/cdb3383a4019da5f5ef0568dfacd542e/.tmp, totalSize=36.5 K 2024-11-24T22:15:03,552 DEBUG [RS:0;bd6592c37bb2:40087-longCompactions-0 {}] compactions.Compactor(225): Compacting 5f46e33e87a64df1ac7d18d27391ff55, keycount=150, bloomtype=ROW, size=12.5 K, encoding=NONE, compression=NONE, seqNum=246, earliestPutTs=1732486497885 2024-11-24T22:15:03,552 DEBUG [RS:0;bd6592c37bb2:40087-shortCompactions-0 {}] compactions.Compactor(225): Compacting 432138b440f142868293ae834c1ed131, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=282, earliestPutTs=1732486500417 2024-11-24T22:15:03,552 DEBUG [RS:0;bd6592c37bb2:40087-longCompactions-0 {}] compactions.Compactor(225): Compacting a35577419c784db99f9973e09dcc1e29, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=282, earliestPutTs=1732486500417 2024-11-24T22:15:03,552 DEBUG [RS:0;bd6592c37bb2:40087-shortCompactions-0 {}] compactions.Compactor(225): Compacting ff0660ba25d7415a932d59d3791e2c25, keycount=200, bloomtype=ROW, size=14.4 K, encoding=NONE, compression=NONE, seqNum=297, earliestPutTs=1732486501601 2024-11-24T22:15:03,553 DEBUG [RS:0;bd6592c37bb2:40087-longCompactions-0 {}] compactions.Compactor(225): Compacting 043730d19ec944f5b2bb93ed6b5bfdbd, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=297, earliestPutTs=1732486501626 2024-11-24T22:15:03,560 INFO [RS:0;bd6592c37bb2:40087-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): cdb3383a4019da5f5ef0568dfacd542e#A#compaction#258 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-24T22:15:03,561 DEBUG [RS:0;bd6592c37bb2:40087-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/cdb3383a4019da5f5ef0568dfacd542e/.tmp/A/9db012e14a6642439d367bf3ad8fe408 is 50, key is test_row_0/A:col10/1732486501635/Put/seqid=0 2024-11-24T22:15:03,563 INFO [RS:0;bd6592c37bb2:40087-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): cdb3383a4019da5f5ef0568dfacd542e#B#compaction#259 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-24T22:15:03,563 DEBUG [RS:0;bd6592c37bb2:40087-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/cdb3383a4019da5f5ef0568dfacd542e/.tmp/B/2cd3c91224164040a16965bf25f89bac is 50, key is test_row_0/B:col10/1732486501635/Put/seqid=0 2024-11-24T22:15:03,567 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45929 is added to blk_1073742133_1309 (size=13017) 2024-11-24T22:15:03,568 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45929 is added to blk_1073742132_1308 (size=13017) 2024-11-24T22:15:03,572 DEBUG [RS:0;bd6592c37bb2:40087-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/cdb3383a4019da5f5ef0568dfacd542e/.tmp/B/2cd3c91224164040a16965bf25f89bac as hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/cdb3383a4019da5f5ef0568dfacd542e/B/2cd3c91224164040a16965bf25f89bac 2024-11-24T22:15:03,576 INFO [RS:0;bd6592c37bb2:40087-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in cdb3383a4019da5f5ef0568dfacd542e/B of cdb3383a4019da5f5ef0568dfacd542e into 2cd3c91224164040a16965bf25f89bac(size=12.7 K), total size for store is 12.7 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-24T22:15:03,576 DEBUG [RS:0;bd6592c37bb2:40087-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for cdb3383a4019da5f5ef0568dfacd542e: 2024-11-24T22:15:03,576 INFO [RS:0;bd6592c37bb2:40087-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732486488674.cdb3383a4019da5f5ef0568dfacd542e., storeName=cdb3383a4019da5f5ef0568dfacd542e/B, priority=13, startTime=1732486503550; duration=0sec 2024-11-24T22:15:03,576 DEBUG [RS:0;bd6592c37bb2:40087-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-24T22:15:03,576 DEBUG [RS:0;bd6592c37bb2:40087-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: cdb3383a4019da5f5ef0568dfacd542e:B 2024-11-24T22:15:03,576 DEBUG [RS:0;bd6592c37bb2:40087-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-24T22:15:03,577 DEBUG [RS:0;bd6592c37bb2:40087-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37367 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-24T22:15:03,577 DEBUG [RS:0;bd6592c37bb2:40087-longCompactions-0 {}] regionserver.HStore(1541): cdb3383a4019da5f5ef0568dfacd542e/C is initiating minor compaction (all files) 2024-11-24T22:15:03,577 INFO [RS:0;bd6592c37bb2:40087-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of cdb3383a4019da5f5ef0568dfacd542e/C in TestAcidGuarantees,,1732486488674.cdb3383a4019da5f5ef0568dfacd542e. 2024-11-24T22:15:03,577 INFO [RS:0;bd6592c37bb2:40087-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/cdb3383a4019da5f5ef0568dfacd542e/C/4a20a25ec25d4e2ab5355cba35fbd701, hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/cdb3383a4019da5f5ef0568dfacd542e/C/59837082e57443849300f8992554f804, hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/cdb3383a4019da5f5ef0568dfacd542e/C/58c7394de6c14021aae54876cf65fa09] into tmpdir=hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/cdb3383a4019da5f5ef0568dfacd542e/.tmp, totalSize=36.5 K 2024-11-24T22:15:03,578 DEBUG [RS:0;bd6592c37bb2:40087-longCompactions-0 {}] compactions.Compactor(225): Compacting 4a20a25ec25d4e2ab5355cba35fbd701, keycount=150, bloomtype=ROW, size=12.5 K, encoding=NONE, compression=NONE, seqNum=246, earliestPutTs=1732486497885 2024-11-24T22:15:03,578 DEBUG [RS:0;bd6592c37bb2:40087-longCompactions-0 {}] compactions.Compactor(225): Compacting 59837082e57443849300f8992554f804, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=282, earliestPutTs=1732486500417 2024-11-24T22:15:03,579 DEBUG [RS:0;bd6592c37bb2:40087-longCompactions-0 {}] compactions.Compactor(225): Compacting 58c7394de6c14021aae54876cf65fa09, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=297, earliestPutTs=1732486501626 2024-11-24T22:15:03,585 INFO [RS:0;bd6592c37bb2:40087-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): cdb3383a4019da5f5ef0568dfacd542e#C#compaction#260 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-24T22:15:03,586 DEBUG [RS:0;bd6592c37bb2:40087-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/cdb3383a4019da5f5ef0568dfacd542e/.tmp/C/e8dc99fbf56546d6a0a6358cb5dbd3d0 is 50, key is test_row_0/C:col10/1732486501635/Put/seqid=0 2024-11-24T22:15:03,590 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45929 is added to blk_1073742134_1310 (size=13017) 2024-11-24T22:15:03,595 DEBUG [RS:0;bd6592c37bb2:40087-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/cdb3383a4019da5f5ef0568dfacd542e/.tmp/C/e8dc99fbf56546d6a0a6358cb5dbd3d0 as hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/cdb3383a4019da5f5ef0568dfacd542e/C/e8dc99fbf56546d6a0a6358cb5dbd3d0 2024-11-24T22:15:03,599 INFO [RS:0;bd6592c37bb2:40087-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in cdb3383a4019da5f5ef0568dfacd542e/C of cdb3383a4019da5f5ef0568dfacd542e into e8dc99fbf56546d6a0a6358cb5dbd3d0(size=12.7 K), total size for store is 12.7 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-24T22:15:03,599 DEBUG [RS:0;bd6592c37bb2:40087-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for cdb3383a4019da5f5ef0568dfacd542e: 2024-11-24T22:15:03,599 INFO [RS:0;bd6592c37bb2:40087-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732486488674.cdb3383a4019da5f5ef0568dfacd542e., storeName=cdb3383a4019da5f5ef0568dfacd542e/C, priority=13, startTime=1732486503550; duration=0sec 2024-11-24T22:15:03,599 DEBUG [RS:0;bd6592c37bb2:40087-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-24T22:15:03,599 DEBUG [RS:0;bd6592c37bb2:40087-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: cdb3383a4019da5f5ef0568dfacd542e:C 2024-11-24T22:15:03,635 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=40087 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=73 2024-11-24T22:15:03,635 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bd6592c37bb2:0-1 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732486488674.cdb3383a4019da5f5ef0568dfacd542e. 2024-11-24T22:15:03,636 INFO [RS_FLUSH_OPERATIONS-regionserver/bd6592c37bb2:0-1 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.HRegion(2902): Flushing cdb3383a4019da5f5ef0568dfacd542e 3/3 column families, dataSize=114.05 KB heapSize=339.42 KB 2024-11-24T22:15:03,636 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bd6592c37bb2:0-1 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.CompactingMemStore(205): FLUSHING TO DISK cdb3383a4019da5f5ef0568dfacd542e, store=A 2024-11-24T22:15:03,636 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bd6592c37bb2:0-1 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-24T22:15:03,636 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bd6592c37bb2:0-1 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.CompactingMemStore(205): FLUSHING TO DISK cdb3383a4019da5f5ef0568dfacd542e, store=B 2024-11-24T22:15:03,636 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bd6592c37bb2:0-1 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-24T22:15:03,636 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bd6592c37bb2:0-1 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.CompactingMemStore(205): FLUSHING TO DISK cdb3383a4019da5f5ef0568dfacd542e, store=C 2024-11-24T22:15:03,636 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bd6592c37bb2:0-1 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-24T22:15:03,641 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bd6592c37bb2:0-1 {event_type=RS_FLUSH_REGIONS, pid=73}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/cdb3383a4019da5f5ef0568dfacd542e/.tmp/A/d5f5c9e33fef41fc8ef8df16567a8fe8 is 50, key is test_row_0/A:col10/1732486502381/Put/seqid=0 2024-11-24T22:15:03,645 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45929 is added to blk_1073742135_1311 (size=12301) 2024-11-24T22:15:03,646 INFO [RS_FLUSH_OPERATIONS-regionserver/bd6592c37bb2:0-1 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=38.02 KB at sequenceid=319 (bloomFilter=true), to=hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/cdb3383a4019da5f5ef0568dfacd542e/.tmp/A/d5f5c9e33fef41fc8ef8df16567a8fe8 2024-11-24T22:15:03,653 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bd6592c37bb2:0-1 {event_type=RS_FLUSH_REGIONS, pid=73}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/cdb3383a4019da5f5ef0568dfacd542e/.tmp/B/93e2d9ce32ef4a27ae31bc67ec3b86a3 is 50, key is test_row_0/B:col10/1732486502381/Put/seqid=0 2024-11-24T22:15:03,667 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45929 is added to blk_1073742136_1312 (size=12301) 2024-11-24T22:15:03,974 DEBUG [RS:0;bd6592c37bb2:40087-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/cdb3383a4019da5f5ef0568dfacd542e/.tmp/A/9db012e14a6642439d367bf3ad8fe408 as hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/cdb3383a4019da5f5ef0568dfacd542e/A/9db012e14a6642439d367bf3ad8fe408 2024-11-24T22:15:03,978 INFO [RS:0;bd6592c37bb2:40087-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in cdb3383a4019da5f5ef0568dfacd542e/A of cdb3383a4019da5f5ef0568dfacd542e into 9db012e14a6642439d367bf3ad8fe408(size=12.7 K), total size for store is 12.7 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-24T22:15:03,978 DEBUG [RS:0;bd6592c37bb2:40087-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for cdb3383a4019da5f5ef0568dfacd542e: 2024-11-24T22:15:03,978 INFO [RS:0;bd6592c37bb2:40087-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732486488674.cdb3383a4019da5f5ef0568dfacd542e., storeName=cdb3383a4019da5f5ef0568dfacd542e/A, priority=13, startTime=1732486503550; duration=0sec 2024-11-24T22:15:03,978 DEBUG [RS:0;bd6592c37bb2:40087-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-24T22:15:03,978 DEBUG [RS:0;bd6592c37bb2:40087-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: cdb3383a4019da5f5ef0568dfacd542e:A 2024-11-24T22:15:04,068 INFO [RS_FLUSH_OPERATIONS-regionserver/bd6592c37bb2:0-1 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=38.02 KB at sequenceid=319 (bloomFilter=true), to=hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/cdb3383a4019da5f5ef0568dfacd542e/.tmp/B/93e2d9ce32ef4a27ae31bc67ec3b86a3 2024-11-24T22:15:04,077 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bd6592c37bb2:0-1 {event_type=RS_FLUSH_REGIONS, pid=73}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/cdb3383a4019da5f5ef0568dfacd542e/.tmp/C/485cf734e750420f950cac587454942a is 50, key is test_row_0/C:col10/1732486502381/Put/seqid=0 2024-11-24T22:15:04,083 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45929 is added to blk_1073742137_1313 (size=12301) 2024-11-24T22:15:04,083 INFO [RS_FLUSH_OPERATIONS-regionserver/bd6592c37bb2:0-1 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=38.02 KB at sequenceid=319 (bloomFilter=true), to=hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/cdb3383a4019da5f5ef0568dfacd542e/.tmp/C/485cf734e750420f950cac587454942a 2024-11-24T22:15:04,087 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bd6592c37bb2:0-1 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/cdb3383a4019da5f5ef0568dfacd542e/.tmp/A/d5f5c9e33fef41fc8ef8df16567a8fe8 as hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/cdb3383a4019da5f5ef0568dfacd542e/A/d5f5c9e33fef41fc8ef8df16567a8fe8 2024-11-24T22:15:04,091 INFO [RS_FLUSH_OPERATIONS-regionserver/bd6592c37bb2:0-1 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/cdb3383a4019da5f5ef0568dfacd542e/A/d5f5c9e33fef41fc8ef8df16567a8fe8, entries=150, sequenceid=319, filesize=12.0 K 2024-11-24T22:15:04,092 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bd6592c37bb2:0-1 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/cdb3383a4019da5f5ef0568dfacd542e/.tmp/B/93e2d9ce32ef4a27ae31bc67ec3b86a3 as hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/cdb3383a4019da5f5ef0568dfacd542e/B/93e2d9ce32ef4a27ae31bc67ec3b86a3 2024-11-24T22:15:04,109 INFO [RS_FLUSH_OPERATIONS-regionserver/bd6592c37bb2:0-1 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/cdb3383a4019da5f5ef0568dfacd542e/B/93e2d9ce32ef4a27ae31bc67ec3b86a3, entries=150, sequenceid=319, filesize=12.0 K 2024-11-24T22:15:04,110 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bd6592c37bb2:0-1 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/cdb3383a4019da5f5ef0568dfacd542e/.tmp/C/485cf734e750420f950cac587454942a as hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/cdb3383a4019da5f5ef0568dfacd542e/C/485cf734e750420f950cac587454942a 2024-11-24T22:15:04,117 INFO [RS_FLUSH_OPERATIONS-regionserver/bd6592c37bb2:0-1 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/cdb3383a4019da5f5ef0568dfacd542e/C/485cf734e750420f950cac587454942a, entries=150, sequenceid=319, filesize=12.0 K 2024-11-24T22:15:04,118 INFO [RS_FLUSH_OPERATIONS-regionserver/bd6592c37bb2:0-1 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.HRegion(3140): Finished flush of dataSize ~114.05 KB/116790, heapSize ~339.38 KB/347520, currentSize=0 B/0 for cdb3383a4019da5f5ef0568dfacd542e in 481ms, sequenceid=319, compaction requested=false 2024-11-24T22:15:04,118 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bd6592c37bb2:0-1 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.HRegion(2603): Flush status journal for cdb3383a4019da5f5ef0568dfacd542e: 2024-11-24T22:15:04,118 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bd6592c37bb2:0-1 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732486488674.cdb3383a4019da5f5ef0568dfacd542e. 2024-11-24T22:15:04,118 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bd6592c37bb2:0-1 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=73 2024-11-24T22:15:04,118 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37701 {}] master.HMaster(4169): Remote procedure done, pid=73 2024-11-24T22:15:04,122 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=73, resume processing ppid=72 2024-11-24T22:15:04,122 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=73, ppid=72, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.8640 sec 2024-11-24T22:15:04,125 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=72, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=72, table=TestAcidGuarantees in 1.8720 sec 2024-11-24T22:15:04,411 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37701 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=72 2024-11-24T22:15:04,411 INFO [RPCClient-NioEventLoopGroup-4-6 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-24T22:15:04,411 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37701 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-24T22:15:04,413 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37701 {}] procedure2.ProcedureExecutor(1139): Stored pid=74, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=74, table=TestAcidGuarantees 2024-11-24T22:15:04,413 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37701 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=74 2024-11-24T22:15:04,414 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=74, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=74, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-24T22:15:04,414 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=74, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=74, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-24T22:15:04,415 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=75, ppid=74, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-24T22:15:04,520 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37701 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=74 2024-11-24T22:15:04,569 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=40087 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=75 2024-11-24T22:15:04,569 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bd6592c37bb2:0-2 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732486488674.cdb3383a4019da5f5ef0568dfacd542e. 2024-11-24T22:15:04,569 INFO [RS_FLUSH_OPERATIONS-regionserver/bd6592c37bb2:0-2 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.HRegion(2902): Flushing cdb3383a4019da5f5ef0568dfacd542e 3/3 column families, dataSize=40.25 KB heapSize=120.28 KB 2024-11-24T22:15:04,569 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40087 {}] regionserver.HRegion(8855): Flush requested on cdb3383a4019da5f5ef0568dfacd542e 2024-11-24T22:15:04,569 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732486488674.cdb3383a4019da5f5ef0568dfacd542e. as already flushing 2024-11-24T22:15:04,570 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bd6592c37bb2:0-2 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.CompactingMemStore(205): FLUSHING TO DISK cdb3383a4019da5f5ef0568dfacd542e, store=A 2024-11-24T22:15:04,570 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bd6592c37bb2:0-2 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-24T22:15:04,570 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bd6592c37bb2:0-2 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.CompactingMemStore(205): FLUSHING TO DISK cdb3383a4019da5f5ef0568dfacd542e, store=B 2024-11-24T22:15:04,570 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bd6592c37bb2:0-2 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-24T22:15:04,570 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bd6592c37bb2:0-2 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.CompactingMemStore(205): FLUSHING TO DISK cdb3383a4019da5f5ef0568dfacd542e, store=C 2024-11-24T22:15:04,570 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bd6592c37bb2:0-2 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-24T22:15:04,575 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bd6592c37bb2:0-2 {event_type=RS_FLUSH_REGIONS, pid=75}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/cdb3383a4019da5f5ef0568dfacd542e/.tmp/A/92e9d617735f4590b5aa7359e2f87d5e is 50, key is test_row_0/A:col10/1732486504563/Put/seqid=0 2024-11-24T22:15:04,590 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45929 is added to blk_1073742138_1314 (size=12301) 2024-11-24T22:15:04,590 INFO [RS_FLUSH_OPERATIONS-regionserver/bd6592c37bb2:0-2 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=330 (bloomFilter=true), to=hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/cdb3383a4019da5f5ef0568dfacd542e/.tmp/A/92e9d617735f4590b5aa7359e2f87d5e 2024-11-24T22:15:04,600 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bd6592c37bb2:0-2 {event_type=RS_FLUSH_REGIONS, pid=75}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/cdb3383a4019da5f5ef0568dfacd542e/.tmp/B/56a31373efde4595be985a3f9afdc70b is 50, key is test_row_0/B:col10/1732486504563/Put/seqid=0 2024-11-24T22:15:04,604 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45929 is added to blk_1073742139_1315 (size=12301) 2024-11-24T22:15:04,676 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/bd6592c37bb2:0-4 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1732486504595, max timestamp=1732486504672Num uniques 859; 2024-11-24T22:15:04,676 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/bd6592c37bb2:0-5 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1732486504595, max timestamp=1732486504672Num uniques 859; 2024-11-24T22:15:04,676 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/bd6592c37bb2:0-7 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1732486504595, max timestamp=1732486504672Num uniques 859; 2024-11-24T22:15:04,676 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/bd6592c37bb2:0-4 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-11-24T22:15:04,676 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/bd6592c37bb2:0-5 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-11-24T22:15:04,676 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/bd6592c37bb2:0-7 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-11-24T22:15:04,730 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37701 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=74 2024-11-24T22:15:04,808 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40087 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cdb3383a4019da5f5ef0568dfacd542e, server=bd6592c37bb2,40087,1732486423713 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-24T22:15:04,808 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40087 {}] ipc.CallRunner(138): callId: 298 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40834 deadline: 1732486564802, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cdb3383a4019da5f5ef0568dfacd542e, server=bd6592c37bb2,40087,1732486423713 2024-11-24T22:15:04,808 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732486488674.cdb3383a4019da5f5ef0568dfacd542e., hostname=bd6592c37bb2,40087,1732486423713, seqNum=2 , the old value is region=TestAcidGuarantees,,1732486488674.cdb3383a4019da5f5ef0568dfacd542e., hostname=bd6592c37bb2,40087,1732486423713, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cdb3383a4019da5f5ef0568dfacd542e, server=bd6592c37bb2,40087,1732486423713 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T22:15:04,808 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732486488674.cdb3383a4019da5f5ef0568dfacd542e., hostname=bd6592c37bb2,40087,1732486423713, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cdb3383a4019da5f5ef0568dfacd542e, server=bd6592c37bb2,40087,1732486423713 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T22:15:04,808 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732486488674.cdb3383a4019da5f5ef0568dfacd542e., hostname=bd6592c37bb2,40087,1732486423713, seqNum=2 because the exception is null or not the one we care about 2024-11-24T22:15:04,915 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40087 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cdb3383a4019da5f5ef0568dfacd542e, server=bd6592c37bb2,40087,1732486423713 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-24T22:15:04,915 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40087 {}] ipc.CallRunner(138): callId: 299 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40834 deadline: 1732486564911, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cdb3383a4019da5f5ef0568dfacd542e, server=bd6592c37bb2,40087,1732486423713 2024-11-24T22:15:04,915 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732486488674.cdb3383a4019da5f5ef0568dfacd542e., hostname=bd6592c37bb2,40087,1732486423713, seqNum=2 , the old value is region=TestAcidGuarantees,,1732486488674.cdb3383a4019da5f5ef0568dfacd542e., hostname=bd6592c37bb2,40087,1732486423713, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cdb3383a4019da5f5ef0568dfacd542e, server=bd6592c37bb2,40087,1732486423713 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T22:15:04,915 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732486488674.cdb3383a4019da5f5ef0568dfacd542e., hostname=bd6592c37bb2,40087,1732486423713, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cdb3383a4019da5f5ef0568dfacd542e, server=bd6592c37bb2,40087,1732486423713 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T22:15:04,916 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732486488674.cdb3383a4019da5f5ef0568dfacd542e., hostname=bd6592c37bb2,40087,1732486423713, seqNum=2 because the exception is null or not the one we care about 2024-11-24T22:15:05,005 INFO [RS_FLUSH_OPERATIONS-regionserver/bd6592c37bb2:0-2 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=330 (bloomFilter=true), to=hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/cdb3383a4019da5f5ef0568dfacd542e/.tmp/B/56a31373efde4595be985a3f9afdc70b 2024-11-24T22:15:05,014 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bd6592c37bb2:0-2 {event_type=RS_FLUSH_REGIONS, pid=75}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/cdb3383a4019da5f5ef0568dfacd542e/.tmp/C/2ea8b80df3ec498ea0b9fa5886e9194a is 50, key is test_row_0/C:col10/1732486504563/Put/seqid=0 2024-11-24T22:15:05,020 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45929 is added to blk_1073742140_1316 (size=12301) 2024-11-24T22:15:05,041 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37701 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=74 2024-11-24T22:15:05,124 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40087 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cdb3383a4019da5f5ef0568dfacd542e, server=bd6592c37bb2,40087,1732486423713 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-24T22:15:05,124 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40087 {}] ipc.CallRunner(138): callId: 300 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40834 deadline: 1732486565121, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cdb3383a4019da5f5ef0568dfacd542e, server=bd6592c37bb2,40087,1732486423713 2024-11-24T22:15:05,125 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732486488674.cdb3383a4019da5f5ef0568dfacd542e., hostname=bd6592c37bb2,40087,1732486423713, seqNum=2 , the old value is region=TestAcidGuarantees,,1732486488674.cdb3383a4019da5f5ef0568dfacd542e., hostname=bd6592c37bb2,40087,1732486423713, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cdb3383a4019da5f5ef0568dfacd542e, server=bd6592c37bb2,40087,1732486423713 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T22:15:05,125 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732486488674.cdb3383a4019da5f5ef0568dfacd542e., hostname=bd6592c37bb2,40087,1732486423713, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cdb3383a4019da5f5ef0568dfacd542e, server=bd6592c37bb2,40087,1732486423713 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T22:15:05,125 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732486488674.cdb3383a4019da5f5ef0568dfacd542e., hostname=bd6592c37bb2,40087,1732486423713, seqNum=2 because the exception is null or not the one we care about 2024-11-24T22:15:05,420 INFO [RS_FLUSH_OPERATIONS-regionserver/bd6592c37bb2:0-2 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=330 (bloomFilter=true), to=hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/cdb3383a4019da5f5ef0568dfacd542e/.tmp/C/2ea8b80df3ec498ea0b9fa5886e9194a 2024-11-24T22:15:05,425 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bd6592c37bb2:0-2 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/cdb3383a4019da5f5ef0568dfacd542e/.tmp/A/92e9d617735f4590b5aa7359e2f87d5e as hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/cdb3383a4019da5f5ef0568dfacd542e/A/92e9d617735f4590b5aa7359e2f87d5e 2024-11-24T22:15:05,430 INFO [RS_FLUSH_OPERATIONS-regionserver/bd6592c37bb2:0-2 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/cdb3383a4019da5f5ef0568dfacd542e/A/92e9d617735f4590b5aa7359e2f87d5e, entries=150, sequenceid=330, filesize=12.0 K 2024-11-24T22:15:05,430 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bd6592c37bb2:0-2 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/cdb3383a4019da5f5ef0568dfacd542e/.tmp/B/56a31373efde4595be985a3f9afdc70b as hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/cdb3383a4019da5f5ef0568dfacd542e/B/56a31373efde4595be985a3f9afdc70b 2024-11-24T22:15:05,433 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40087 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cdb3383a4019da5f5ef0568dfacd542e, server=bd6592c37bb2,40087,1732486423713 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-24T22:15:05,433 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40087 {}] ipc.CallRunner(138): callId: 301 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40834 deadline: 1732486565431, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cdb3383a4019da5f5ef0568dfacd542e, server=bd6592c37bb2,40087,1732486423713 2024-11-24T22:15:05,434 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732486488674.cdb3383a4019da5f5ef0568dfacd542e., hostname=bd6592c37bb2,40087,1732486423713, seqNum=2 , the old value is region=TestAcidGuarantees,,1732486488674.cdb3383a4019da5f5ef0568dfacd542e., hostname=bd6592c37bb2,40087,1732486423713, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cdb3383a4019da5f5ef0568dfacd542e, server=bd6592c37bb2,40087,1732486423713 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T22:15:05,434 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732486488674.cdb3383a4019da5f5ef0568dfacd542e., hostname=bd6592c37bb2,40087,1732486423713, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cdb3383a4019da5f5ef0568dfacd542e, server=bd6592c37bb2,40087,1732486423713 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T22:15:05,434 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732486488674.cdb3383a4019da5f5ef0568dfacd542e., hostname=bd6592c37bb2,40087,1732486423713, seqNum=2 because the exception is null or not the one we care about 2024-11-24T22:15:05,434 INFO [RS_FLUSH_OPERATIONS-regionserver/bd6592c37bb2:0-2 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/cdb3383a4019da5f5ef0568dfacd542e/B/56a31373efde4595be985a3f9afdc70b, entries=150, sequenceid=330, filesize=12.0 K 2024-11-24T22:15:05,435 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bd6592c37bb2:0-2 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/cdb3383a4019da5f5ef0568dfacd542e/.tmp/C/2ea8b80df3ec498ea0b9fa5886e9194a as hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/cdb3383a4019da5f5ef0568dfacd542e/C/2ea8b80df3ec498ea0b9fa5886e9194a 2024-11-24T22:15:05,440 INFO [RS_FLUSH_OPERATIONS-regionserver/bd6592c37bb2:0-2 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/cdb3383a4019da5f5ef0568dfacd542e/C/2ea8b80df3ec498ea0b9fa5886e9194a, entries=150, sequenceid=330, filesize=12.0 K 2024-11-24T22:15:05,441 INFO [RS_FLUSH_OPERATIONS-regionserver/bd6592c37bb2:0-2 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.HRegion(3140): Finished flush of dataSize ~46.96 KB/48090, heapSize ~140.16 KB/143520, currentSize=134.18 KB/137400 for cdb3383a4019da5f5ef0568dfacd542e in 872ms, sequenceid=330, compaction requested=true 2024-11-24T22:15:05,441 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bd6592c37bb2:0-2 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.HRegion(2603): Flush status journal for cdb3383a4019da5f5ef0568dfacd542e: 2024-11-24T22:15:05,441 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bd6592c37bb2:0-2 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732486488674.cdb3383a4019da5f5ef0568dfacd542e. 2024-11-24T22:15:05,441 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bd6592c37bb2:0-2 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=75 2024-11-24T22:15:05,442 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37701 {}] master.HMaster(4169): Remote procedure done, pid=75 2024-11-24T22:15:05,446 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=75, resume processing ppid=74 2024-11-24T22:15:05,446 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=75, ppid=74, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.0300 sec 2024-11-24T22:15:05,447 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=74, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=74, table=TestAcidGuarantees in 1.0340 sec 2024-11-24T22:15:05,551 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37701 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=74 2024-11-24T22:15:05,551 INFO [RPCClient-NioEventLoopGroup-4-6 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-24T22:15:05,552 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37701 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-24T22:15:05,552 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37701 {}] procedure2.ProcedureExecutor(1139): Stored pid=76, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=76, table=TestAcidGuarantees 2024-11-24T22:15:05,553 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37701 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=76 2024-11-24T22:15:05,553 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=76, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=76, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-24T22:15:05,554 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=76, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=76, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-24T22:15:05,554 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=77, ppid=76, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-24T22:15:05,661 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37701 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=76 2024-11-24T22:15:05,706 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=40087 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=77 2024-11-24T22:15:05,707 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bd6592c37bb2:0-0 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732486488674.cdb3383a4019da5f5ef0568dfacd542e. 2024-11-24T22:15:05,707 INFO [RS_FLUSH_OPERATIONS-regionserver/bd6592c37bb2:0-0 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.HRegion(2902): Flushing cdb3383a4019da5f5ef0568dfacd542e 3/3 column families, dataSize=134.18 KB heapSize=384.98 KB 2024-11-24T22:15:05,707 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bd6592c37bb2:0-0 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.CompactingMemStore(205): FLUSHING TO DISK cdb3383a4019da5f5ef0568dfacd542e, store=A 2024-11-24T22:15:05,707 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bd6592c37bb2:0-0 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-24T22:15:05,707 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bd6592c37bb2:0-0 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.CompactingMemStore(205): FLUSHING TO DISK cdb3383a4019da5f5ef0568dfacd542e, store=B 2024-11-24T22:15:05,707 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bd6592c37bb2:0-0 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-24T22:15:05,707 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bd6592c37bb2:0-0 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.CompactingMemStore(205): FLUSHING TO DISK cdb3383a4019da5f5ef0568dfacd542e, store=C 2024-11-24T22:15:05,707 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bd6592c37bb2:0-0 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-24T22:15:05,713 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bd6592c37bb2:0-0 {event_type=RS_FLUSH_REGIONS, pid=77}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/cdb3383a4019da5f5ef0568dfacd542e/.tmp/A/9134c9ad987e430695cb0d7651a28586 is 50, key is test_row_0/A:col10/1732486504736/Put/seqid=0 2024-11-24T22:15:05,718 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45929 is added to blk_1073742141_1317 (size=12301) 2024-11-24T22:15:05,870 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37701 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=76 2024-11-24T22:15:05,945 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40087 {}] regionserver.HRegion(8855): Flush requested on cdb3383a4019da5f5ef0568dfacd542e 2024-11-24T22:15:05,945 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732486488674.cdb3383a4019da5f5ef0568dfacd542e. as already flushing 2024-11-24T22:15:05,983 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40087 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cdb3383a4019da5f5ef0568dfacd542e, server=bd6592c37bb2,40087,1732486423713 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-24T22:15:05,983 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40087 {}] ipc.CallRunner(138): callId: 309 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40834 deadline: 1732486565978, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cdb3383a4019da5f5ef0568dfacd542e, server=bd6592c37bb2,40087,1732486423713 2024-11-24T22:15:05,983 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732486488674.cdb3383a4019da5f5ef0568dfacd542e., hostname=bd6592c37bb2,40087,1732486423713, seqNum=2 , the old value is region=TestAcidGuarantees,,1732486488674.cdb3383a4019da5f5ef0568dfacd542e., hostname=bd6592c37bb2,40087,1732486423713, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cdb3383a4019da5f5ef0568dfacd542e, server=bd6592c37bb2,40087,1732486423713 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T22:15:05,983 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732486488674.cdb3383a4019da5f5ef0568dfacd542e., hostname=bd6592c37bb2,40087,1732486423713, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cdb3383a4019da5f5ef0568dfacd542e, server=bd6592c37bb2,40087,1732486423713 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T22:15:05,983 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732486488674.cdb3383a4019da5f5ef0568dfacd542e., hostname=bd6592c37bb2,40087,1732486423713, seqNum=2 because the exception is null or not the one we care about 2024-11-24T22:15:06,092 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40087 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cdb3383a4019da5f5ef0568dfacd542e, server=bd6592c37bb2,40087,1732486423713 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-24T22:15:06,092 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40087 {}] ipc.CallRunner(138): callId: 310 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40834 deadline: 1732486566091, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cdb3383a4019da5f5ef0568dfacd542e, server=bd6592c37bb2,40087,1732486423713 2024-11-24T22:15:06,092 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732486488674.cdb3383a4019da5f5ef0568dfacd542e., hostname=bd6592c37bb2,40087,1732486423713, seqNum=2 , the old value is region=TestAcidGuarantees,,1732486488674.cdb3383a4019da5f5ef0568dfacd542e., hostname=bd6592c37bb2,40087,1732486423713, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cdb3383a4019da5f5ef0568dfacd542e, server=bd6592c37bb2,40087,1732486423713 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T22:15:06,092 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732486488674.cdb3383a4019da5f5ef0568dfacd542e., hostname=bd6592c37bb2,40087,1732486423713, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cdb3383a4019da5f5ef0568dfacd542e, server=bd6592c37bb2,40087,1732486423713 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T22:15:06,092 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732486488674.cdb3383a4019da5f5ef0568dfacd542e., hostname=bd6592c37bb2,40087,1732486423713, seqNum=2 because the exception is null or not the one we care about 2024-11-24T22:15:06,119 INFO [RS_FLUSH_OPERATIONS-regionserver/bd6592c37bb2:0-0 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=367 (bloomFilter=true), to=hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/cdb3383a4019da5f5ef0568dfacd542e/.tmp/A/9134c9ad987e430695cb0d7651a28586 2024-11-24T22:15:06,132 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bd6592c37bb2:0-0 {event_type=RS_FLUSH_REGIONS, pid=77}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/cdb3383a4019da5f5ef0568dfacd542e/.tmp/B/ef3421fddb9a422b990293a78aab8059 is 50, key is test_row_0/B:col10/1732486504736/Put/seqid=0 2024-11-24T22:15:06,138 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45929 is added to blk_1073742142_1318 (size=12301) 2024-11-24T22:15:06,181 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37701 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=76 2024-11-24T22:15:06,306 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40087 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cdb3383a4019da5f5ef0568dfacd542e, server=bd6592c37bb2,40087,1732486423713 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-24T22:15:06,306 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40087 {}] ipc.CallRunner(138): callId: 311 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40834 deadline: 1732486566301, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cdb3383a4019da5f5ef0568dfacd542e, server=bd6592c37bb2,40087,1732486423713 2024-11-24T22:15:06,306 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732486488674.cdb3383a4019da5f5ef0568dfacd542e., hostname=bd6592c37bb2,40087,1732486423713, seqNum=2 , the old value is region=TestAcidGuarantees,,1732486488674.cdb3383a4019da5f5ef0568dfacd542e., hostname=bd6592c37bb2,40087,1732486423713, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cdb3383a4019da5f5ef0568dfacd542e, server=bd6592c37bb2,40087,1732486423713 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T22:15:06,306 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732486488674.cdb3383a4019da5f5ef0568dfacd542e., hostname=bd6592c37bb2,40087,1732486423713, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cdb3383a4019da5f5ef0568dfacd542e, server=bd6592c37bb2,40087,1732486423713 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T22:15:06,306 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732486488674.cdb3383a4019da5f5ef0568dfacd542e., hostname=bd6592c37bb2,40087,1732486423713, seqNum=2 because the exception is null or not the one we care about 2024-11-24T22:15:06,539 INFO [RS_FLUSH_OPERATIONS-regionserver/bd6592c37bb2:0-0 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=367 (bloomFilter=true), to=hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/cdb3383a4019da5f5ef0568dfacd542e/.tmp/B/ef3421fddb9a422b990293a78aab8059 2024-11-24T22:15:06,547 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bd6592c37bb2:0-0 {event_type=RS_FLUSH_REGIONS, pid=77}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/cdb3383a4019da5f5ef0568dfacd542e/.tmp/C/0defd1196d7f48eab2401586ed1b976d is 50, key is test_row_0/C:col10/1732486504736/Put/seqid=0 2024-11-24T22:15:06,551 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45929 is added to blk_1073742143_1319 (size=12301) 2024-11-24T22:15:06,552 INFO [RS_FLUSH_OPERATIONS-regionserver/bd6592c37bb2:0-0 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=367 (bloomFilter=true), to=hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/cdb3383a4019da5f5ef0568dfacd542e/.tmp/C/0defd1196d7f48eab2401586ed1b976d 2024-11-24T22:15:06,556 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bd6592c37bb2:0-0 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/cdb3383a4019da5f5ef0568dfacd542e/.tmp/A/9134c9ad987e430695cb0d7651a28586 as hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/cdb3383a4019da5f5ef0568dfacd542e/A/9134c9ad987e430695cb0d7651a28586 2024-11-24T22:15:06,560 INFO [RS_FLUSH_OPERATIONS-regionserver/bd6592c37bb2:0-0 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/cdb3383a4019da5f5ef0568dfacd542e/A/9134c9ad987e430695cb0d7651a28586, entries=150, sequenceid=367, filesize=12.0 K 2024-11-24T22:15:06,560 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bd6592c37bb2:0-0 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/cdb3383a4019da5f5ef0568dfacd542e/.tmp/B/ef3421fddb9a422b990293a78aab8059 as hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/cdb3383a4019da5f5ef0568dfacd542e/B/ef3421fddb9a422b990293a78aab8059 2024-11-24T22:15:06,564 INFO [RS_FLUSH_OPERATIONS-regionserver/bd6592c37bb2:0-0 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/cdb3383a4019da5f5ef0568dfacd542e/B/ef3421fddb9a422b990293a78aab8059, entries=150, sequenceid=367, filesize=12.0 K 2024-11-24T22:15:06,565 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bd6592c37bb2:0-0 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/cdb3383a4019da5f5ef0568dfacd542e/.tmp/C/0defd1196d7f48eab2401586ed1b976d as hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/cdb3383a4019da5f5ef0568dfacd542e/C/0defd1196d7f48eab2401586ed1b976d 2024-11-24T22:15:06,570 INFO [RS_FLUSH_OPERATIONS-regionserver/bd6592c37bb2:0-0 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/cdb3383a4019da5f5ef0568dfacd542e/C/0defd1196d7f48eab2401586ed1b976d, entries=150, sequenceid=367, filesize=12.0 K 2024-11-24T22:15:06,571 INFO [RS_FLUSH_OPERATIONS-regionserver/bd6592c37bb2:0-0 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.HRegion(3140): Finished flush of dataSize ~134.18 KB/137400, heapSize ~384.94 KB/394176, currentSize=46.96 KB/48090 for cdb3383a4019da5f5ef0568dfacd542e in 864ms, sequenceid=367, compaction requested=true 2024-11-24T22:15:06,571 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bd6592c37bb2:0-0 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.HRegion(2603): Flush status journal for cdb3383a4019da5f5ef0568dfacd542e: 2024-11-24T22:15:06,571 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bd6592c37bb2:0-0 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732486488674.cdb3383a4019da5f5ef0568dfacd542e. 2024-11-24T22:15:06,571 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bd6592c37bb2:0-0 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=77 2024-11-24T22:15:06,571 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37701 {}] master.HMaster(4169): Remote procedure done, pid=77 2024-11-24T22:15:06,575 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=77, resume processing ppid=76 2024-11-24T22:15:06,575 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=77, ppid=76, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.0180 sec 2024-11-24T22:15:06,577 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=76, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=76, table=TestAcidGuarantees in 1.0240 sec 2024-11-24T22:15:06,614 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40087 {}] regionserver.HRegion(8855): Flush requested on cdb3383a4019da5f5ef0568dfacd542e 2024-11-24T22:15:06,614 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing cdb3383a4019da5f5ef0568dfacd542e 3/3 column families, dataSize=53.67 KB heapSize=160.13 KB 2024-11-24T22:15:06,614 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK cdb3383a4019da5f5ef0568dfacd542e, store=A 2024-11-24T22:15:06,614 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-24T22:15:06,614 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK cdb3383a4019da5f5ef0568dfacd542e, store=B 2024-11-24T22:15:06,614 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-24T22:15:06,614 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK cdb3383a4019da5f5ef0568dfacd542e, store=C 2024-11-24T22:15:06,615 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-24T22:15:06,619 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/cdb3383a4019da5f5ef0568dfacd542e/.tmp/A/738c0933918f47e68f1f385ea5ede3d9 is 50, key is test_row_0/A:col10/1732486505965/Put/seqid=0 2024-11-24T22:15:06,624 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45929 is added to blk_1073742144_1320 (size=14741) 2024-11-24T22:15:06,625 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=378 (bloomFilter=true), to=hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/cdb3383a4019da5f5ef0568dfacd542e/.tmp/A/738c0933918f47e68f1f385ea5ede3d9 2024-11-24T22:15:06,633 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/cdb3383a4019da5f5ef0568dfacd542e/.tmp/B/8508b162a0b54202bd7b981aca6af1c6 is 50, key is test_row_0/B:col10/1732486505965/Put/seqid=0 2024-11-24T22:15:06,646 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45929 is added to blk_1073742145_1321 (size=12301) 2024-11-24T22:15:06,691 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37701 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=76 2024-11-24T22:15:06,691 INFO [RPCClient-NioEventLoopGroup-4-6 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-24T22:15:06,691 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37701 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-24T22:15:06,692 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37701 {}] procedure2.ProcedureExecutor(1139): Stored pid=78, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=78, table=TestAcidGuarantees 2024-11-24T22:15:06,693 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37701 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=78 2024-11-24T22:15:06,693 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=78, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=78, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-24T22:15:06,693 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=78, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=78, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-24T22:15:06,694 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=79, ppid=78, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-24T22:15:06,725 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/bd6592c37bb2:0-9 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1732486506702, max timestamp=1732486506721Num uniques 859; 2024-11-24T22:15:06,725 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/bd6592c37bb2:0-0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1732486506702, max timestamp=1732486506721Num uniques 859; 2024-11-24T22:15:06,725 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/bd6592c37bb2:0-9 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-11-24T22:15:06,725 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/bd6592c37bb2:0-0 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-11-24T22:15:06,725 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/bd6592c37bb2:0-8 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1732486506702, max timestamp=1732486506721Num uniques 859; 2024-11-24T22:15:06,725 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/bd6592c37bb2:0-8 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-11-24T22:15:06,801 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37701 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=78 2024-11-24T22:15:06,836 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40087 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cdb3383a4019da5f5ef0568dfacd542e, server=bd6592c37bb2,40087,1732486423713 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-24T22:15:06,837 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40087 {}] ipc.CallRunner(138): callId: 346 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40834 deadline: 1732486566830, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cdb3383a4019da5f5ef0568dfacd542e, server=bd6592c37bb2,40087,1732486423713 2024-11-24T22:15:06,837 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732486488674.cdb3383a4019da5f5ef0568dfacd542e., hostname=bd6592c37bb2,40087,1732486423713, seqNum=2 , the old value is region=TestAcidGuarantees,,1732486488674.cdb3383a4019da5f5ef0568dfacd542e., hostname=bd6592c37bb2,40087,1732486423713, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cdb3383a4019da5f5ef0568dfacd542e, server=bd6592c37bb2,40087,1732486423713 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T22:15:06,837 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732486488674.cdb3383a4019da5f5ef0568dfacd542e., hostname=bd6592c37bb2,40087,1732486423713, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cdb3383a4019da5f5ef0568dfacd542e, server=bd6592c37bb2,40087,1732486423713 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T22:15:06,837 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732486488674.cdb3383a4019da5f5ef0568dfacd542e., hostname=bd6592c37bb2,40087,1732486423713, seqNum=2 because the exception is null or not the one we care about 2024-11-24T22:15:06,846 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=40087 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=79 2024-11-24T22:15:06,846 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bd6592c37bb2:0-1 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732486488674.cdb3383a4019da5f5ef0568dfacd542e. 2024-11-24T22:15:06,846 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bd6592c37bb2:0-1 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732486488674.cdb3383a4019da5f5ef0568dfacd542e. as already flushing 2024-11-24T22:15:06,846 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bd6592c37bb2:0-1 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732486488674.cdb3383a4019da5f5ef0568dfacd542e. 2024-11-24T22:15:06,846 ERROR [RS_FLUSH_OPERATIONS-regionserver/bd6592c37bb2:0-1 {event_type=RS_FLUSH_REGIONS, pid=79}] handler.RSProcedureHandler(58): pid=79 java.io.IOException: Unable to complete flush {ENCODED => cdb3383a4019da5f5ef0568dfacd542e, NAME => 'TestAcidGuarantees,,1732486488674.cdb3383a4019da5f5ef0568dfacd542e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T22:15:06,846 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bd6592c37bb2:0-1 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=79 java.io.IOException: Unable to complete flush {ENCODED => cdb3383a4019da5f5ef0568dfacd542e, NAME => 'TestAcidGuarantees,,1732486488674.cdb3383a4019da5f5ef0568dfacd542e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T22:15:06,847 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37701 {}] master.HMaster(4177): Remote procedure failed, pid=79 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => cdb3383a4019da5f5ef0568dfacd542e, NAME => 'TestAcidGuarantees,,1732486488674.cdb3383a4019da5f5ef0568dfacd542e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => cdb3383a4019da5f5ef0568dfacd542e, NAME => 'TestAcidGuarantees,,1732486488674.cdb3383a4019da5f5ef0568dfacd542e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T22:15:06,944 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40087 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cdb3383a4019da5f5ef0568dfacd542e, server=bd6592c37bb2,40087,1732486423713 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-24T22:15:06,944 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40087 {}] ipc.CallRunner(138): callId: 347 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40834 deadline: 1732486566941, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cdb3383a4019da5f5ef0568dfacd542e, server=bd6592c37bb2,40087,1732486423713 2024-11-24T22:15:06,945 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732486488674.cdb3383a4019da5f5ef0568dfacd542e., hostname=bd6592c37bb2,40087,1732486423713, seqNum=2 , the old value is region=TestAcidGuarantees,,1732486488674.cdb3383a4019da5f5ef0568dfacd542e., hostname=bd6592c37bb2,40087,1732486423713, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cdb3383a4019da5f5ef0568dfacd542e, server=bd6592c37bb2,40087,1732486423713 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T22:15:06,945 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732486488674.cdb3383a4019da5f5ef0568dfacd542e., hostname=bd6592c37bb2,40087,1732486423713, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cdb3383a4019da5f5ef0568dfacd542e, server=bd6592c37bb2,40087,1732486423713 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T22:15:06,945 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732486488674.cdb3383a4019da5f5ef0568dfacd542e., hostname=bd6592c37bb2,40087,1732486423713, seqNum=2 because the exception is null or not the one we care about 2024-11-24T22:15:06,999 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=40087 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=79 2024-11-24T22:15:06,999 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bd6592c37bb2:0-2 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732486488674.cdb3383a4019da5f5ef0568dfacd542e. 2024-11-24T22:15:06,999 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bd6592c37bb2:0-2 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732486488674.cdb3383a4019da5f5ef0568dfacd542e. as already flushing 2024-11-24T22:15:06,999 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bd6592c37bb2:0-2 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732486488674.cdb3383a4019da5f5ef0568dfacd542e. 2024-11-24T22:15:06,999 ERROR [RS_FLUSH_OPERATIONS-regionserver/bd6592c37bb2:0-2 {event_type=RS_FLUSH_REGIONS, pid=79}] handler.RSProcedureHandler(58): pid=79 java.io.IOException: Unable to complete flush {ENCODED => cdb3383a4019da5f5ef0568dfacd542e, NAME => 'TestAcidGuarantees,,1732486488674.cdb3383a4019da5f5ef0568dfacd542e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T22:15:06,999 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bd6592c37bb2:0-2 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=79 java.io.IOException: Unable to complete flush {ENCODED => cdb3383a4019da5f5ef0568dfacd542e, NAME => 'TestAcidGuarantees,,1732486488674.cdb3383a4019da5f5ef0568dfacd542e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T22:15:07,000 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37701 {}] master.HMaster(4177): Remote procedure failed, pid=79 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => cdb3383a4019da5f5ef0568dfacd542e, NAME => 'TestAcidGuarantees,,1732486488674.cdb3383a4019da5f5ef0568dfacd542e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => cdb3383a4019da5f5ef0568dfacd542e, NAME => 'TestAcidGuarantees,,1732486488674.cdb3383a4019da5f5ef0568dfacd542e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T22:15:07,010 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37701 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=78 2024-11-24T22:15:07,047 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=378 (bloomFilter=true), to=hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/cdb3383a4019da5f5ef0568dfacd542e/.tmp/B/8508b162a0b54202bd7b981aca6af1c6 2024-11-24T22:15:07,055 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/cdb3383a4019da5f5ef0568dfacd542e/.tmp/C/441c2523681548fa9d09028a3484fa21 is 50, key is test_row_0/C:col10/1732486505965/Put/seqid=0 2024-11-24T22:15:07,059 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45929 is added to blk_1073742146_1322 (size=12301) 2024-11-24T22:15:07,060 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=378 (bloomFilter=true), to=hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/cdb3383a4019da5f5ef0568dfacd542e/.tmp/C/441c2523681548fa9d09028a3484fa21 2024-11-24T22:15:07,065 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/cdb3383a4019da5f5ef0568dfacd542e/.tmp/A/738c0933918f47e68f1f385ea5ede3d9 as hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/cdb3383a4019da5f5ef0568dfacd542e/A/738c0933918f47e68f1f385ea5ede3d9 2024-11-24T22:15:07,068 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/cdb3383a4019da5f5ef0568dfacd542e/A/738c0933918f47e68f1f385ea5ede3d9, entries=200, sequenceid=378, filesize=14.4 K 2024-11-24T22:15:07,069 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/cdb3383a4019da5f5ef0568dfacd542e/.tmp/B/8508b162a0b54202bd7b981aca6af1c6 as hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/cdb3383a4019da5f5ef0568dfacd542e/B/8508b162a0b54202bd7b981aca6af1c6 2024-11-24T22:15:07,073 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/cdb3383a4019da5f5ef0568dfacd542e/B/8508b162a0b54202bd7b981aca6af1c6, entries=150, sequenceid=378, filesize=12.0 K 2024-11-24T22:15:07,074 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/cdb3383a4019da5f5ef0568dfacd542e/.tmp/C/441c2523681548fa9d09028a3484fa21 as hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/cdb3383a4019da5f5ef0568dfacd542e/C/441c2523681548fa9d09028a3484fa21 2024-11-24T22:15:07,080 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/cdb3383a4019da5f5ef0568dfacd542e/C/441c2523681548fa9d09028a3484fa21, entries=150, sequenceid=378, filesize=12.0 K 2024-11-24T22:15:07,081 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~53.67 KB/54960, heapSize ~160.08 KB/163920, currentSize=127.47 KB/130530 for cdb3383a4019da5f5ef0568dfacd542e in 467ms, sequenceid=378, compaction requested=true 2024-11-24T22:15:07,081 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for cdb3383a4019da5f5ef0568dfacd542e: 2024-11-24T22:15:07,081 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store cdb3383a4019da5f5ef0568dfacd542e:A, priority=-2147483648, current under compaction store size is 1 2024-11-24T22:15:07,081 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-24T22:15:07,081 DEBUG [RS:0;bd6592c37bb2:40087-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 5 store files, 0 compacting, 5 eligible, 16 blocking 2024-11-24T22:15:07,081 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store cdb3383a4019da5f5ef0568dfacd542e:B, priority=-2147483648, current under compaction store size is 2 2024-11-24T22:15:07,081 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-24T22:15:07,081 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store cdb3383a4019da5f5ef0568dfacd542e:C, priority=-2147483648, current under compaction store size is 3 2024-11-24T22:15:07,081 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-24T22:15:07,081 DEBUG [RS:0;bd6592c37bb2:40087-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 5 store files, 0 compacting, 5 eligible, 16 blocking 2024-11-24T22:15:07,083 DEBUG [RS:0;bd6592c37bb2:40087-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 5 files of size 64661 starting at candidate #0 after considering 6 permutations with 6 in ratio 2024-11-24T22:15:07,083 DEBUG [RS:0;bd6592c37bb2:40087-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 5 files of size 62221 starting at candidate #0 after considering 6 permutations with 6 in ratio 2024-11-24T22:15:07,083 DEBUG [RS:0;bd6592c37bb2:40087-shortCompactions-0 {}] regionserver.HStore(1541): cdb3383a4019da5f5ef0568dfacd542e/A is initiating minor compaction (all files) 2024-11-24T22:15:07,083 DEBUG [RS:0;bd6592c37bb2:40087-longCompactions-0 {}] regionserver.HStore(1541): cdb3383a4019da5f5ef0568dfacd542e/B is initiating minor compaction (all files) 2024-11-24T22:15:07,083 INFO [RS:0;bd6592c37bb2:40087-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of cdb3383a4019da5f5ef0568dfacd542e/A in TestAcidGuarantees,,1732486488674.cdb3383a4019da5f5ef0568dfacd542e. 2024-11-24T22:15:07,083 INFO [RS:0;bd6592c37bb2:40087-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of cdb3383a4019da5f5ef0568dfacd542e/B in TestAcidGuarantees,,1732486488674.cdb3383a4019da5f5ef0568dfacd542e. 2024-11-24T22:15:07,083 INFO [RS:0;bd6592c37bb2:40087-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/cdb3383a4019da5f5ef0568dfacd542e/A/9db012e14a6642439d367bf3ad8fe408, hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/cdb3383a4019da5f5ef0568dfacd542e/A/d5f5c9e33fef41fc8ef8df16567a8fe8, hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/cdb3383a4019da5f5ef0568dfacd542e/A/92e9d617735f4590b5aa7359e2f87d5e, hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/cdb3383a4019da5f5ef0568dfacd542e/A/9134c9ad987e430695cb0d7651a28586, hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/cdb3383a4019da5f5ef0568dfacd542e/A/738c0933918f47e68f1f385ea5ede3d9] into tmpdir=hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/cdb3383a4019da5f5ef0568dfacd542e/.tmp, totalSize=63.1 K 2024-11-24T22:15:07,083 INFO [RS:0;bd6592c37bb2:40087-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/cdb3383a4019da5f5ef0568dfacd542e/B/2cd3c91224164040a16965bf25f89bac, hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/cdb3383a4019da5f5ef0568dfacd542e/B/93e2d9ce32ef4a27ae31bc67ec3b86a3, hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/cdb3383a4019da5f5ef0568dfacd542e/B/56a31373efde4595be985a3f9afdc70b, hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/cdb3383a4019da5f5ef0568dfacd542e/B/ef3421fddb9a422b990293a78aab8059, hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/cdb3383a4019da5f5ef0568dfacd542e/B/8508b162a0b54202bd7b981aca6af1c6] into tmpdir=hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/cdb3383a4019da5f5ef0568dfacd542e/.tmp, totalSize=60.8 K 2024-11-24T22:15:07,083 DEBUG [RS:0;bd6592c37bb2:40087-shortCompactions-0 {}] compactions.Compactor(225): Compacting 9db012e14a6642439d367bf3ad8fe408, keycount=150, bloomtype=ROW, size=12.7 K, encoding=NONE, compression=NONE, seqNum=297, earliestPutTs=1732486501626 2024-11-24T22:15:07,083 DEBUG [RS:0;bd6592c37bb2:40087-longCompactions-0 {}] compactions.Compactor(225): Compacting 2cd3c91224164040a16965bf25f89bac, keycount=150, bloomtype=ROW, size=12.7 K, encoding=NONE, compression=NONE, seqNum=297, earliestPutTs=1732486501626 2024-11-24T22:15:07,083 DEBUG [RS:0;bd6592c37bb2:40087-longCompactions-0 {}] compactions.Compactor(225): Compacting 93e2d9ce32ef4a27ae31bc67ec3b86a3, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=319, earliestPutTs=1732486502374 2024-11-24T22:15:07,084 DEBUG [RS:0;bd6592c37bb2:40087-shortCompactions-0 {}] compactions.Compactor(225): Compacting d5f5c9e33fef41fc8ef8df16567a8fe8, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=319, earliestPutTs=1732486502374 2024-11-24T22:15:07,084 DEBUG [RS:0;bd6592c37bb2:40087-longCompactions-0 {}] compactions.Compactor(225): Compacting 56a31373efde4595be985a3f9afdc70b, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=330, earliestPutTs=1732486504563 2024-11-24T22:15:07,084 DEBUG [RS:0;bd6592c37bb2:40087-shortCompactions-0 {}] compactions.Compactor(225): Compacting 92e9d617735f4590b5aa7359e2f87d5e, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=330, earliestPutTs=1732486504563 2024-11-24T22:15:07,084 DEBUG [RS:0;bd6592c37bb2:40087-longCompactions-0 {}] compactions.Compactor(225): Compacting ef3421fddb9a422b990293a78aab8059, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=367, earliestPutTs=1732486504736 2024-11-24T22:15:07,084 DEBUG [RS:0;bd6592c37bb2:40087-shortCompactions-0 {}] compactions.Compactor(225): Compacting 9134c9ad987e430695cb0d7651a28586, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=367, earliestPutTs=1732486504736 2024-11-24T22:15:07,084 DEBUG [RS:0;bd6592c37bb2:40087-longCompactions-0 {}] compactions.Compactor(225): Compacting 8508b162a0b54202bd7b981aca6af1c6, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=378, earliestPutTs=1732486505960 2024-11-24T22:15:07,085 DEBUG [RS:0;bd6592c37bb2:40087-shortCompactions-0 {}] compactions.Compactor(225): Compacting 738c0933918f47e68f1f385ea5ede3d9, keycount=200, bloomtype=ROW, size=14.4 K, encoding=NONE, compression=NONE, seqNum=378, earliestPutTs=1732486505960 2024-11-24T22:15:07,100 INFO [RS:0;bd6592c37bb2:40087-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): cdb3383a4019da5f5ef0568dfacd542e#A#compaction#273 average throughput is 2.18 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-24T22:15:07,101 DEBUG [RS:0;bd6592c37bb2:40087-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/cdb3383a4019da5f5ef0568dfacd542e/.tmp/A/110c91754e0a45988a2018e87fa8d232 is 50, key is test_row_0/A:col10/1732486505965/Put/seqid=0 2024-11-24T22:15:07,116 INFO [RS:0;bd6592c37bb2:40087-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): cdb3383a4019da5f5ef0568dfacd542e#B#compaction#274 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-24T22:15:07,116 DEBUG [RS:0;bd6592c37bb2:40087-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/cdb3383a4019da5f5ef0568dfacd542e/.tmp/B/e94895b2f12149d9bfc0802c0a2e7d36 is 50, key is test_row_0/B:col10/1732486505965/Put/seqid=0 2024-11-24T22:15:07,127 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45929 is added to blk_1073742147_1323 (size=13187) 2024-11-24T22:15:07,136 DEBUG [RS:0;bd6592c37bb2:40087-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/cdb3383a4019da5f5ef0568dfacd542e/.tmp/A/110c91754e0a45988a2018e87fa8d232 as hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/cdb3383a4019da5f5ef0568dfacd542e/A/110c91754e0a45988a2018e87fa8d232 2024-11-24T22:15:07,142 INFO [RS:0;bd6592c37bb2:40087-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 5 (all) file(s) in cdb3383a4019da5f5ef0568dfacd542e/A of cdb3383a4019da5f5ef0568dfacd542e into 110c91754e0a45988a2018e87fa8d232(size=12.9 K), total size for store is 12.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-24T22:15:07,144 DEBUG [RS:0;bd6592c37bb2:40087-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for cdb3383a4019da5f5ef0568dfacd542e: 2024-11-24T22:15:07,144 INFO [RS:0;bd6592c37bb2:40087-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732486488674.cdb3383a4019da5f5ef0568dfacd542e., storeName=cdb3383a4019da5f5ef0568dfacd542e/A, priority=11, startTime=1732486507081; duration=0sec 2024-11-24T22:15:07,144 DEBUG [RS:0;bd6592c37bb2:40087-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-24T22:15:07,144 DEBUG [RS:0;bd6592c37bb2:40087-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: cdb3383a4019da5f5ef0568dfacd542e:A 2024-11-24T22:15:07,144 DEBUG [RS:0;bd6592c37bb2:40087-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 5 store files, 0 compacting, 5 eligible, 16 blocking 2024-11-24T22:15:07,147 DEBUG [RS:0;bd6592c37bb2:40087-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 5 files of size 62221 starting at candidate #0 after considering 6 permutations with 6 in ratio 2024-11-24T22:15:07,147 DEBUG [RS:0;bd6592c37bb2:40087-shortCompactions-0 {}] regionserver.HStore(1541): cdb3383a4019da5f5ef0568dfacd542e/C is initiating minor compaction (all files) 2024-11-24T22:15:07,148 INFO [RS:0;bd6592c37bb2:40087-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of cdb3383a4019da5f5ef0568dfacd542e/C in TestAcidGuarantees,,1732486488674.cdb3383a4019da5f5ef0568dfacd542e. 2024-11-24T22:15:07,148 INFO [RS:0;bd6592c37bb2:40087-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/cdb3383a4019da5f5ef0568dfacd542e/C/e8dc99fbf56546d6a0a6358cb5dbd3d0, hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/cdb3383a4019da5f5ef0568dfacd542e/C/485cf734e750420f950cac587454942a, hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/cdb3383a4019da5f5ef0568dfacd542e/C/2ea8b80df3ec498ea0b9fa5886e9194a, hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/cdb3383a4019da5f5ef0568dfacd542e/C/0defd1196d7f48eab2401586ed1b976d, hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/cdb3383a4019da5f5ef0568dfacd542e/C/441c2523681548fa9d09028a3484fa21] into tmpdir=hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/cdb3383a4019da5f5ef0568dfacd542e/.tmp, totalSize=60.8 K 2024-11-24T22:15:07,148 DEBUG [RS:0;bd6592c37bb2:40087-shortCompactions-0 {}] compactions.Compactor(225): Compacting e8dc99fbf56546d6a0a6358cb5dbd3d0, keycount=150, bloomtype=ROW, size=12.7 K, encoding=NONE, compression=NONE, seqNum=297, earliestPutTs=1732486501626 2024-11-24T22:15:07,148 DEBUG [RS:0;bd6592c37bb2:40087-shortCompactions-0 {}] compactions.Compactor(225): Compacting 485cf734e750420f950cac587454942a, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=319, earliestPutTs=1732486502374 2024-11-24T22:15:07,149 DEBUG [RS:0;bd6592c37bb2:40087-shortCompactions-0 {}] compactions.Compactor(225): Compacting 2ea8b80df3ec498ea0b9fa5886e9194a, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=330, earliestPutTs=1732486504563 2024-11-24T22:15:07,149 DEBUG [RS:0;bd6592c37bb2:40087-shortCompactions-0 {}] compactions.Compactor(225): Compacting 0defd1196d7f48eab2401586ed1b976d, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=367, earliestPutTs=1732486504736 2024-11-24T22:15:07,150 DEBUG [RS:0;bd6592c37bb2:40087-shortCompactions-0 {}] compactions.Compactor(225): Compacting 441c2523681548fa9d09028a3484fa21, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=378, earliestPutTs=1732486505960 2024-11-24T22:15:07,152 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=40087 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=79 2024-11-24T22:15:07,152 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bd6592c37bb2:0-0 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732486488674.cdb3383a4019da5f5ef0568dfacd542e. 2024-11-24T22:15:07,152 INFO [RS_FLUSH_OPERATIONS-regionserver/bd6592c37bb2:0-0 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.HRegion(2902): Flushing cdb3383a4019da5f5ef0568dfacd542e 3/3 column families, dataSize=127.47 KB heapSize=365.06 KB 2024-11-24T22:15:07,153 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bd6592c37bb2:0-0 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.CompactingMemStore(205): FLUSHING TO DISK cdb3383a4019da5f5ef0568dfacd542e, store=A 2024-11-24T22:15:07,153 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bd6592c37bb2:0-0 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-24T22:15:07,153 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bd6592c37bb2:0-0 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.CompactingMemStore(205): FLUSHING TO DISK cdb3383a4019da5f5ef0568dfacd542e, store=B 2024-11-24T22:15:07,153 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bd6592c37bb2:0-0 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-24T22:15:07,153 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bd6592c37bb2:0-0 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.CompactingMemStore(205): FLUSHING TO DISK cdb3383a4019da5f5ef0568dfacd542e, store=C 2024-11-24T22:15:07,153 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bd6592c37bb2:0-0 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-24T22:15:07,155 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40087 {}] regionserver.HRegion(8855): Flush requested on cdb3383a4019da5f5ef0568dfacd542e 2024-11-24T22:15:07,156 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732486488674.cdb3383a4019da5f5ef0568dfacd542e. as already flushing 2024-11-24T22:15:07,167 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bd6592c37bb2:0-0 {event_type=RS_FLUSH_REGIONS, pid=79}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/cdb3383a4019da5f5ef0568dfacd542e/.tmp/A/ea40fb6886784ae982e1a5b7960a5111 is 50, key is test_row_0/A:col10/1732486506821/Put/seqid=0 2024-11-24T22:15:07,169 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45929 is added to blk_1073742148_1324 (size=13187) 2024-11-24T22:15:07,178 DEBUG [RS:0;bd6592c37bb2:40087-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/cdb3383a4019da5f5ef0568dfacd542e/.tmp/B/e94895b2f12149d9bfc0802c0a2e7d36 as hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/cdb3383a4019da5f5ef0568dfacd542e/B/e94895b2f12149d9bfc0802c0a2e7d36 2024-11-24T22:15:07,188 INFO [RS:0;bd6592c37bb2:40087-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 5 (all) file(s) in cdb3383a4019da5f5ef0568dfacd542e/B of cdb3383a4019da5f5ef0568dfacd542e into e94895b2f12149d9bfc0802c0a2e7d36(size=12.9 K), total size for store is 12.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-24T22:15:07,188 DEBUG [RS:0;bd6592c37bb2:40087-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for cdb3383a4019da5f5ef0568dfacd542e: 2024-11-24T22:15:07,188 INFO [RS:0;bd6592c37bb2:40087-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732486488674.cdb3383a4019da5f5ef0568dfacd542e., storeName=cdb3383a4019da5f5ef0568dfacd542e/B, priority=11, startTime=1732486507081; duration=0sec 2024-11-24T22:15:07,188 DEBUG [RS:0;bd6592c37bb2:40087-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-24T22:15:07,188 DEBUG [RS:0;bd6592c37bb2:40087-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: cdb3383a4019da5f5ef0568dfacd542e:B 2024-11-24T22:15:07,200 INFO [RS:0;bd6592c37bb2:40087-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): cdb3383a4019da5f5ef0568dfacd542e#C#compaction#276 average throughput is 2.18 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-24T22:15:07,201 DEBUG [RS:0;bd6592c37bb2:40087-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/cdb3383a4019da5f5ef0568dfacd542e/.tmp/C/f4856e4fded1464d906bb6d736463799 is 50, key is test_row_0/C:col10/1732486505965/Put/seqid=0 2024-11-24T22:15:07,213 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45929 is added to blk_1073742149_1325 (size=12301) 2024-11-24T22:15:07,214 INFO [RS_FLUSH_OPERATIONS-regionserver/bd6592c37bb2:0-0 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=42.49 KB at sequenceid=415 (bloomFilter=true), to=hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/cdb3383a4019da5f5ef0568dfacd542e/.tmp/A/ea40fb6886784ae982e1a5b7960a5111 2024-11-24T22:15:07,223 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40087 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cdb3383a4019da5f5ef0568dfacd542e, server=bd6592c37bb2,40087,1732486423713 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-24T22:15:07,224 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40087 {}] ipc.CallRunner(138): callId: 356 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40834 deadline: 1732486567218, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cdb3383a4019da5f5ef0568dfacd542e, server=bd6592c37bb2,40087,1732486423713 2024-11-24T22:15:07,224 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732486488674.cdb3383a4019da5f5ef0568dfacd542e., hostname=bd6592c37bb2,40087,1732486423713, seqNum=2 , the old value is region=TestAcidGuarantees,,1732486488674.cdb3383a4019da5f5ef0568dfacd542e., hostname=bd6592c37bb2,40087,1732486423713, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cdb3383a4019da5f5ef0568dfacd542e, server=bd6592c37bb2,40087,1732486423713 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T22:15:07,224 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732486488674.cdb3383a4019da5f5ef0568dfacd542e., hostname=bd6592c37bb2,40087,1732486423713, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cdb3383a4019da5f5ef0568dfacd542e, server=bd6592c37bb2,40087,1732486423713 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T22:15:07,224 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732486488674.cdb3383a4019da5f5ef0568dfacd542e., hostname=bd6592c37bb2,40087,1732486423713, seqNum=2 because the exception is null or not the one we care about 2024-11-24T22:15:07,230 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45929 is added to blk_1073742150_1326 (size=13187) 2024-11-24T22:15:07,238 DEBUG [RS:0;bd6592c37bb2:40087-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/cdb3383a4019da5f5ef0568dfacd542e/.tmp/C/f4856e4fded1464d906bb6d736463799 as hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/cdb3383a4019da5f5ef0568dfacd542e/C/f4856e4fded1464d906bb6d736463799 2024-11-24T22:15:07,241 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bd6592c37bb2:0-0 {event_type=RS_FLUSH_REGIONS, pid=79}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/cdb3383a4019da5f5ef0568dfacd542e/.tmp/B/572e0b339dc94be0b36f8ea4a0ef7518 is 50, key is test_row_0/B:col10/1732486506821/Put/seqid=0 2024-11-24T22:15:07,243 INFO [RS:0;bd6592c37bb2:40087-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 5 (all) file(s) in cdb3383a4019da5f5ef0568dfacd542e/C of cdb3383a4019da5f5ef0568dfacd542e into f4856e4fded1464d906bb6d736463799(size=12.9 K), total size for store is 12.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-24T22:15:07,243 DEBUG [RS:0;bd6592c37bb2:40087-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for cdb3383a4019da5f5ef0568dfacd542e: 2024-11-24T22:15:07,243 INFO [RS:0;bd6592c37bb2:40087-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732486488674.cdb3383a4019da5f5ef0568dfacd542e., storeName=cdb3383a4019da5f5ef0568dfacd542e/C, priority=11, startTime=1732486507081; duration=0sec 2024-11-24T22:15:07,243 DEBUG [RS:0;bd6592c37bb2:40087-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-24T22:15:07,243 DEBUG [RS:0;bd6592c37bb2:40087-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: cdb3383a4019da5f5ef0568dfacd542e:C 2024-11-24T22:15:07,258 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45929 is added to blk_1073742151_1327 (size=12301) 2024-11-24T22:15:07,262 INFO [RS_FLUSH_OPERATIONS-regionserver/bd6592c37bb2:0-0 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=42.49 KB at sequenceid=415 (bloomFilter=true), to=hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/cdb3383a4019da5f5ef0568dfacd542e/.tmp/B/572e0b339dc94be0b36f8ea4a0ef7518 2024-11-24T22:15:07,279 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bd6592c37bb2:0-0 {event_type=RS_FLUSH_REGIONS, pid=79}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/cdb3383a4019da5f5ef0568dfacd542e/.tmp/C/845e21e625844395ad910b7963d0e3fc is 50, key is test_row_0/C:col10/1732486506821/Put/seqid=0 2024-11-24T22:15:07,314 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45929 is added to blk_1073742152_1328 (size=12301) 2024-11-24T22:15:07,316 INFO [RS_FLUSH_OPERATIONS-regionserver/bd6592c37bb2:0-0 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=42.49 KB at sequenceid=415 (bloomFilter=true), to=hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/cdb3383a4019da5f5ef0568dfacd542e/.tmp/C/845e21e625844395ad910b7963d0e3fc 2024-11-24T22:15:07,321 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37701 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=78 2024-11-24T22:15:07,322 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bd6592c37bb2:0-0 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/cdb3383a4019da5f5ef0568dfacd542e/.tmp/A/ea40fb6886784ae982e1a5b7960a5111 as hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/cdb3383a4019da5f5ef0568dfacd542e/A/ea40fb6886784ae982e1a5b7960a5111 2024-11-24T22:15:07,330 INFO [RS_FLUSH_OPERATIONS-regionserver/bd6592c37bb2:0-0 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/cdb3383a4019da5f5ef0568dfacd542e/A/ea40fb6886784ae982e1a5b7960a5111, entries=150, sequenceid=415, filesize=12.0 K 2024-11-24T22:15:07,331 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bd6592c37bb2:0-0 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/cdb3383a4019da5f5ef0568dfacd542e/.tmp/B/572e0b339dc94be0b36f8ea4a0ef7518 as hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/cdb3383a4019da5f5ef0568dfacd542e/B/572e0b339dc94be0b36f8ea4a0ef7518 2024-11-24T22:15:07,337 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40087 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cdb3383a4019da5f5ef0568dfacd542e, server=bd6592c37bb2,40087,1732486423713 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-24T22:15:07,337 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40087 {}] ipc.CallRunner(138): callId: 357 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40834 deadline: 1732486567331, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cdb3383a4019da5f5ef0568dfacd542e, server=bd6592c37bb2,40087,1732486423713 2024-11-24T22:15:07,338 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732486488674.cdb3383a4019da5f5ef0568dfacd542e., hostname=bd6592c37bb2,40087,1732486423713, seqNum=2 , the old value is region=TestAcidGuarantees,,1732486488674.cdb3383a4019da5f5ef0568dfacd542e., hostname=bd6592c37bb2,40087,1732486423713, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cdb3383a4019da5f5ef0568dfacd542e, server=bd6592c37bb2,40087,1732486423713 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T22:15:07,338 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732486488674.cdb3383a4019da5f5ef0568dfacd542e., hostname=bd6592c37bb2,40087,1732486423713, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cdb3383a4019da5f5ef0568dfacd542e, server=bd6592c37bb2,40087,1732486423713 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T22:15:07,338 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732486488674.cdb3383a4019da5f5ef0568dfacd542e., hostname=bd6592c37bb2,40087,1732486423713, seqNum=2 because the exception is null or not the one we care about 2024-11-24T22:15:07,339 INFO [RS_FLUSH_OPERATIONS-regionserver/bd6592c37bb2:0-0 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/cdb3383a4019da5f5ef0568dfacd542e/B/572e0b339dc94be0b36f8ea4a0ef7518, entries=150, sequenceid=415, filesize=12.0 K 2024-11-24T22:15:07,340 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bd6592c37bb2:0-0 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/cdb3383a4019da5f5ef0568dfacd542e/.tmp/C/845e21e625844395ad910b7963d0e3fc as hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/cdb3383a4019da5f5ef0568dfacd542e/C/845e21e625844395ad910b7963d0e3fc 2024-11-24T22:15:07,344 INFO [RS_FLUSH_OPERATIONS-regionserver/bd6592c37bb2:0-0 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/cdb3383a4019da5f5ef0568dfacd542e/C/845e21e625844395ad910b7963d0e3fc, entries=150, sequenceid=415, filesize=12.0 K 2024-11-24T22:15:07,345 INFO [RS_FLUSH_OPERATIONS-regionserver/bd6592c37bb2:0-0 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.HRegion(3140): Finished flush of dataSize ~127.47 KB/130530, heapSize ~365.02 KB/373776, currentSize=53.67 KB/54960 for cdb3383a4019da5f5ef0568dfacd542e in 193ms, sequenceid=415, compaction requested=false 2024-11-24T22:15:07,346 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bd6592c37bb2:0-0 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.HRegion(2603): Flush status journal for cdb3383a4019da5f5ef0568dfacd542e: 2024-11-24T22:15:07,346 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bd6592c37bb2:0-0 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732486488674.cdb3383a4019da5f5ef0568dfacd542e. 2024-11-24T22:15:07,346 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bd6592c37bb2:0-0 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=79 2024-11-24T22:15:07,346 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37701 {}] master.HMaster(4169): Remote procedure done, pid=79 2024-11-24T22:15:07,352 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=79, resume processing ppid=78 2024-11-24T22:15:07,352 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=79, ppid=78, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 655 msec 2024-11-24T22:15:07,354 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=78, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=78, table=TestAcidGuarantees in 662 msec 2024-11-24T22:15:07,543 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40087 {}] regionserver.HRegion(8855): Flush requested on cdb3383a4019da5f5ef0568dfacd542e 2024-11-24T22:15:07,543 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing cdb3383a4019da5f5ef0568dfacd542e 3/3 column families, dataSize=60.38 KB heapSize=180.05 KB 2024-11-24T22:15:07,543 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK cdb3383a4019da5f5ef0568dfacd542e, store=A 2024-11-24T22:15:07,543 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-24T22:15:07,543 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK cdb3383a4019da5f5ef0568dfacd542e, store=B 2024-11-24T22:15:07,543 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-24T22:15:07,543 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK cdb3383a4019da5f5ef0568dfacd542e, store=C 2024-11-24T22:15:07,543 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-24T22:15:07,549 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/cdb3383a4019da5f5ef0568dfacd542e/.tmp/A/0518f72a50f24932b904aa7048e69b43 is 50, key is test_row_0/A:col10/1732486507215/Put/seqid=0 2024-11-24T22:15:07,557 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45929 is added to blk_1073742153_1329 (size=12301) 2024-11-24T22:15:07,632 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40087 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cdb3383a4019da5f5ef0568dfacd542e, server=bd6592c37bb2,40087,1732486423713 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-24T22:15:07,633 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40087 {}] ipc.CallRunner(138): callId: 376 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40834 deadline: 1732486567629, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cdb3383a4019da5f5ef0568dfacd542e, server=bd6592c37bb2,40087,1732486423713 2024-11-24T22:15:07,633 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732486488674.cdb3383a4019da5f5ef0568dfacd542e., hostname=bd6592c37bb2,40087,1732486423713, seqNum=2 , the old value is region=TestAcidGuarantees,,1732486488674.cdb3383a4019da5f5ef0568dfacd542e., hostname=bd6592c37bb2,40087,1732486423713, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cdb3383a4019da5f5ef0568dfacd542e, server=bd6592c37bb2,40087,1732486423713 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T22:15:07,633 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732486488674.cdb3383a4019da5f5ef0568dfacd542e., hostname=bd6592c37bb2,40087,1732486423713, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cdb3383a4019da5f5ef0568dfacd542e, server=bd6592c37bb2,40087,1732486423713 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T22:15:07,633 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732486488674.cdb3383a4019da5f5ef0568dfacd542e., hostname=bd6592c37bb2,40087,1732486423713, seqNum=2 because the exception is null or not the one we care about 2024-11-24T22:15:07,743 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40087 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cdb3383a4019da5f5ef0568dfacd542e, server=bd6592c37bb2,40087,1732486423713 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-24T22:15:07,743 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40087 {}] ipc.CallRunner(138): callId: 377 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40834 deadline: 1732486567740, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cdb3383a4019da5f5ef0568dfacd542e, server=bd6592c37bb2,40087,1732486423713 2024-11-24T22:15:07,744 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732486488674.cdb3383a4019da5f5ef0568dfacd542e., hostname=bd6592c37bb2,40087,1732486423713, seqNum=2 , the old value is region=TestAcidGuarantees,,1732486488674.cdb3383a4019da5f5ef0568dfacd542e., hostname=bd6592c37bb2,40087,1732486423713, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cdb3383a4019da5f5ef0568dfacd542e, server=bd6592c37bb2,40087,1732486423713 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T22:15:07,744 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732486488674.cdb3383a4019da5f5ef0568dfacd542e., hostname=bd6592c37bb2,40087,1732486423713, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cdb3383a4019da5f5ef0568dfacd542e, server=bd6592c37bb2,40087,1732486423713 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T22:15:07,744 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732486488674.cdb3383a4019da5f5ef0568dfacd542e., hostname=bd6592c37bb2,40087,1732486423713, seqNum=2 because the exception is null or not the one we care about 2024-11-24T22:15:07,831 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37701 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=78 2024-11-24T22:15:07,831 INFO [RPCClient-NioEventLoopGroup-4-6 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-24T22:15:07,832 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37701 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-24T22:15:07,833 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37701 {}] procedure2.ProcedureExecutor(1139): Stored pid=80, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=80, table=TestAcidGuarantees 2024-11-24T22:15:07,833 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37701 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=80 2024-11-24T22:15:07,834 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=80, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=80, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-24T22:15:07,835 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=80, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=80, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-24T22:15:07,835 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=81, ppid=80, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-24T22:15:07,941 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37701 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=80 2024-11-24T22:15:07,953 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40087 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cdb3383a4019da5f5ef0568dfacd542e, server=bd6592c37bb2,40087,1732486423713 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-24T22:15:07,954 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40087 {}] ipc.CallRunner(138): callId: 378 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40834 deadline: 1732486567951, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cdb3383a4019da5f5ef0568dfacd542e, server=bd6592c37bb2,40087,1732486423713 2024-11-24T22:15:07,954 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732486488674.cdb3383a4019da5f5ef0568dfacd542e., hostname=bd6592c37bb2,40087,1732486423713, seqNum=2 , the old value is region=TestAcidGuarantees,,1732486488674.cdb3383a4019da5f5ef0568dfacd542e., hostname=bd6592c37bb2,40087,1732486423713, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cdb3383a4019da5f5ef0568dfacd542e, server=bd6592c37bb2,40087,1732486423713 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T22:15:07,954 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732486488674.cdb3383a4019da5f5ef0568dfacd542e., hostname=bd6592c37bb2,40087,1732486423713, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cdb3383a4019da5f5ef0568dfacd542e, server=bd6592c37bb2,40087,1732486423713 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T22:15:07,954 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732486488674.cdb3383a4019da5f5ef0568dfacd542e., hostname=bd6592c37bb2,40087,1732486423713, seqNum=2 because the exception is null or not the one we care about 2024-11-24T22:15:07,957 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=429 (bloomFilter=true), to=hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/cdb3383a4019da5f5ef0568dfacd542e/.tmp/A/0518f72a50f24932b904aa7048e69b43 2024-11-24T22:15:07,965 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/cdb3383a4019da5f5ef0568dfacd542e/.tmp/B/2126a18a8f2f46ba8d5ad912a1e84862 is 50, key is test_row_0/B:col10/1732486507215/Put/seqid=0 2024-11-24T22:15:07,969 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45929 is added to blk_1073742154_1330 (size=12301) 2024-11-24T22:15:07,987 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=40087 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=81 2024-11-24T22:15:07,987 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bd6592c37bb2:0-1 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732486488674.cdb3383a4019da5f5ef0568dfacd542e. 2024-11-24T22:15:07,987 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bd6592c37bb2:0-1 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732486488674.cdb3383a4019da5f5ef0568dfacd542e. as already flushing 2024-11-24T22:15:07,987 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bd6592c37bb2:0-1 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732486488674.cdb3383a4019da5f5ef0568dfacd542e. 2024-11-24T22:15:07,987 ERROR [RS_FLUSH_OPERATIONS-regionserver/bd6592c37bb2:0-1 {event_type=RS_FLUSH_REGIONS, pid=81}] handler.RSProcedureHandler(58): pid=81 java.io.IOException: Unable to complete flush {ENCODED => cdb3383a4019da5f5ef0568dfacd542e, NAME => 'TestAcidGuarantees,,1732486488674.cdb3383a4019da5f5ef0568dfacd542e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T22:15:07,987 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bd6592c37bb2:0-1 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=81 java.io.IOException: Unable to complete flush {ENCODED => cdb3383a4019da5f5ef0568dfacd542e, NAME => 'TestAcidGuarantees,,1732486488674.cdb3383a4019da5f5ef0568dfacd542e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T22:15:07,988 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37701 {}] master.HMaster(4177): Remote procedure failed, pid=81 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => cdb3383a4019da5f5ef0568dfacd542e, NAME => 'TestAcidGuarantees,,1732486488674.cdb3383a4019da5f5ef0568dfacd542e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => cdb3383a4019da5f5ef0568dfacd542e, NAME => 'TestAcidGuarantees,,1732486488674.cdb3383a4019da5f5ef0568dfacd542e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T22:15:08,139 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=40087 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=81 2024-11-24T22:15:08,140 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bd6592c37bb2:0-2 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732486488674.cdb3383a4019da5f5ef0568dfacd542e. 2024-11-24T22:15:08,140 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bd6592c37bb2:0-2 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732486488674.cdb3383a4019da5f5ef0568dfacd542e. as already flushing 2024-11-24T22:15:08,140 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bd6592c37bb2:0-2 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732486488674.cdb3383a4019da5f5ef0568dfacd542e. 2024-11-24T22:15:08,140 ERROR [RS_FLUSH_OPERATIONS-regionserver/bd6592c37bb2:0-2 {event_type=RS_FLUSH_REGIONS, pid=81}] handler.RSProcedureHandler(58): pid=81 java.io.IOException: Unable to complete flush {ENCODED => cdb3383a4019da5f5ef0568dfacd542e, NAME => 'TestAcidGuarantees,,1732486488674.cdb3383a4019da5f5ef0568dfacd542e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T22:15:08,140 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bd6592c37bb2:0-2 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=81 java.io.IOException: Unable to complete flush {ENCODED => cdb3383a4019da5f5ef0568dfacd542e, NAME => 'TestAcidGuarantees,,1732486488674.cdb3383a4019da5f5ef0568dfacd542e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T22:15:08,140 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37701 {}] master.HMaster(4177): Remote procedure failed, pid=81 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => cdb3383a4019da5f5ef0568dfacd542e, NAME => 'TestAcidGuarantees,,1732486488674.cdb3383a4019da5f5ef0568dfacd542e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => cdb3383a4019da5f5ef0568dfacd542e, NAME => 'TestAcidGuarantees,,1732486488674.cdb3383a4019da5f5ef0568dfacd542e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T22:15:08,151 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37701 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=80 2024-11-24T22:15:08,271 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40087 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cdb3383a4019da5f5ef0568dfacd542e, server=bd6592c37bb2,40087,1732486423713 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-24T22:15:08,271 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40087 {}] ipc.CallRunner(138): callId: 379 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40834 deadline: 1732486568260, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cdb3383a4019da5f5ef0568dfacd542e, server=bd6592c37bb2,40087,1732486423713 2024-11-24T22:15:08,272 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732486488674.cdb3383a4019da5f5ef0568dfacd542e., hostname=bd6592c37bb2,40087,1732486423713, seqNum=2 , the old value is region=TestAcidGuarantees,,1732486488674.cdb3383a4019da5f5ef0568dfacd542e., hostname=bd6592c37bb2,40087,1732486423713, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cdb3383a4019da5f5ef0568dfacd542e, server=bd6592c37bb2,40087,1732486423713 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T22:15:08,272 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732486488674.cdb3383a4019da5f5ef0568dfacd542e., hostname=bd6592c37bb2,40087,1732486423713, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cdb3383a4019da5f5ef0568dfacd542e, server=bd6592c37bb2,40087,1732486423713 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T22:15:08,272 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732486488674.cdb3383a4019da5f5ef0568dfacd542e., hostname=bd6592c37bb2,40087,1732486423713, seqNum=2 because the exception is null or not the one we care about 2024-11-24T22:15:08,292 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=40087 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=81 2024-11-24T22:15:08,293 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bd6592c37bb2:0-0 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732486488674.cdb3383a4019da5f5ef0568dfacd542e. 2024-11-24T22:15:08,293 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bd6592c37bb2:0-0 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732486488674.cdb3383a4019da5f5ef0568dfacd542e. as already flushing 2024-11-24T22:15:08,293 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bd6592c37bb2:0-0 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732486488674.cdb3383a4019da5f5ef0568dfacd542e. 2024-11-24T22:15:08,293 ERROR [RS_FLUSH_OPERATIONS-regionserver/bd6592c37bb2:0-0 {event_type=RS_FLUSH_REGIONS, pid=81}] handler.RSProcedureHandler(58): pid=81 java.io.IOException: Unable to complete flush {ENCODED => cdb3383a4019da5f5ef0568dfacd542e, NAME => 'TestAcidGuarantees,,1732486488674.cdb3383a4019da5f5ef0568dfacd542e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T22:15:08,293 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bd6592c37bb2:0-0 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=81 java.io.IOException: Unable to complete flush {ENCODED => cdb3383a4019da5f5ef0568dfacd542e, NAME => 'TestAcidGuarantees,,1732486488674.cdb3383a4019da5f5ef0568dfacd542e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T22:15:08,293 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37701 {}] master.HMaster(4177): Remote procedure failed, pid=81 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => cdb3383a4019da5f5ef0568dfacd542e, NAME => 'TestAcidGuarantees,,1732486488674.cdb3383a4019da5f5ef0568dfacd542e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => cdb3383a4019da5f5ef0568dfacd542e, NAME => 'TestAcidGuarantees,,1732486488674.cdb3383a4019da5f5ef0568dfacd542e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T22:15:08,370 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=429 (bloomFilter=true), to=hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/cdb3383a4019da5f5ef0568dfacd542e/.tmp/B/2126a18a8f2f46ba8d5ad912a1e84862 2024-11-24T22:15:08,377 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/cdb3383a4019da5f5ef0568dfacd542e/.tmp/C/fe3bc641923a4531b63d749f303ae574 is 50, key is test_row_0/C:col10/1732486507215/Put/seqid=0 2024-11-24T22:15:08,381 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45929 is added to blk_1073742155_1331 (size=12301) 2024-11-24T22:15:08,445 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=40087 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=81 2024-11-24T22:15:08,445 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bd6592c37bb2:0-1 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732486488674.cdb3383a4019da5f5ef0568dfacd542e. 2024-11-24T22:15:08,445 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bd6592c37bb2:0-1 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732486488674.cdb3383a4019da5f5ef0568dfacd542e. as already flushing 2024-11-24T22:15:08,445 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bd6592c37bb2:0-1 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732486488674.cdb3383a4019da5f5ef0568dfacd542e. 2024-11-24T22:15:08,445 ERROR [RS_FLUSH_OPERATIONS-regionserver/bd6592c37bb2:0-1 {event_type=RS_FLUSH_REGIONS, pid=81}] handler.RSProcedureHandler(58): pid=81 java.io.IOException: Unable to complete flush {ENCODED => cdb3383a4019da5f5ef0568dfacd542e, NAME => 'TestAcidGuarantees,,1732486488674.cdb3383a4019da5f5ef0568dfacd542e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T22:15:08,446 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bd6592c37bb2:0-1 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=81 java.io.IOException: Unable to complete flush {ENCODED => cdb3383a4019da5f5ef0568dfacd542e, NAME => 'TestAcidGuarantees,,1732486488674.cdb3383a4019da5f5ef0568dfacd542e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T22:15:08,446 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37701 {}] master.HMaster(4177): Remote procedure failed, pid=81 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => cdb3383a4019da5f5ef0568dfacd542e, NAME => 'TestAcidGuarantees,,1732486488674.cdb3383a4019da5f5ef0568dfacd542e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => cdb3383a4019da5f5ef0568dfacd542e, NAME => 'TestAcidGuarantees,,1732486488674.cdb3383a4019da5f5ef0568dfacd542e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T22:15:08,461 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37701 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=80 2024-11-24T22:15:08,598 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=40087 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=81 2024-11-24T22:15:08,598 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bd6592c37bb2:0-2 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732486488674.cdb3383a4019da5f5ef0568dfacd542e. 2024-11-24T22:15:08,599 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bd6592c37bb2:0-2 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732486488674.cdb3383a4019da5f5ef0568dfacd542e. as already flushing 2024-11-24T22:15:08,599 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bd6592c37bb2:0-2 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732486488674.cdb3383a4019da5f5ef0568dfacd542e. 2024-11-24T22:15:08,599 ERROR [RS_FLUSH_OPERATIONS-regionserver/bd6592c37bb2:0-2 {event_type=RS_FLUSH_REGIONS, pid=81}] handler.RSProcedureHandler(58): pid=81 java.io.IOException: Unable to complete flush {ENCODED => cdb3383a4019da5f5ef0568dfacd542e, NAME => 'TestAcidGuarantees,,1732486488674.cdb3383a4019da5f5ef0568dfacd542e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T22:15:08,599 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bd6592c37bb2:0-2 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=81 java.io.IOException: Unable to complete flush {ENCODED => cdb3383a4019da5f5ef0568dfacd542e, NAME => 'TestAcidGuarantees,,1732486488674.cdb3383a4019da5f5ef0568dfacd542e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T22:15:08,599 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37701 {}] master.HMaster(4177): Remote procedure failed, pid=81 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => cdb3383a4019da5f5ef0568dfacd542e, NAME => 'TestAcidGuarantees,,1732486488674.cdb3383a4019da5f5ef0568dfacd542e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => cdb3383a4019da5f5ef0568dfacd542e, NAME => 'TestAcidGuarantees,,1732486488674.cdb3383a4019da5f5ef0568dfacd542e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T22:15:08,751 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=40087 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=81 2024-11-24T22:15:08,751 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bd6592c37bb2:0-0 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732486488674.cdb3383a4019da5f5ef0568dfacd542e. 2024-11-24T22:15:08,752 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bd6592c37bb2:0-0 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732486488674.cdb3383a4019da5f5ef0568dfacd542e. as already flushing 2024-11-24T22:15:08,752 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bd6592c37bb2:0-0 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732486488674.cdb3383a4019da5f5ef0568dfacd542e. 2024-11-24T22:15:08,752 ERROR [RS_FLUSH_OPERATIONS-regionserver/bd6592c37bb2:0-0 {event_type=RS_FLUSH_REGIONS, pid=81}] handler.RSProcedureHandler(58): pid=81 java.io.IOException: Unable to complete flush {ENCODED => cdb3383a4019da5f5ef0568dfacd542e, NAME => 'TestAcidGuarantees,,1732486488674.cdb3383a4019da5f5ef0568dfacd542e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T22:15:08,752 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bd6592c37bb2:0-0 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=81 java.io.IOException: Unable to complete flush {ENCODED => cdb3383a4019da5f5ef0568dfacd542e, NAME => 'TestAcidGuarantees,,1732486488674.cdb3383a4019da5f5ef0568dfacd542e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T22:15:08,752 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37701 {}] master.HMaster(4177): Remote procedure failed, pid=81 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => cdb3383a4019da5f5ef0568dfacd542e, NAME => 'TestAcidGuarantees,,1732486488674.cdb3383a4019da5f5ef0568dfacd542e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => cdb3383a4019da5f5ef0568dfacd542e, NAME => 'TestAcidGuarantees,,1732486488674.cdb3383a4019da5f5ef0568dfacd542e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T22:15:08,782 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=429 (bloomFilter=true), to=hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/cdb3383a4019da5f5ef0568dfacd542e/.tmp/C/fe3bc641923a4531b63d749f303ae574 2024-11-24T22:15:08,784 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40087 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cdb3383a4019da5f5ef0568dfacd542e, server=bd6592c37bb2,40087,1732486423713 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-24T22:15:08,784 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40087 {}] ipc.CallRunner(138): callId: 380 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40834 deadline: 1732486568780, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cdb3383a4019da5f5ef0568dfacd542e, server=bd6592c37bb2,40087,1732486423713 2024-11-24T22:15:08,784 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732486488674.cdb3383a4019da5f5ef0568dfacd542e., hostname=bd6592c37bb2,40087,1732486423713, seqNum=2 , the old value is region=TestAcidGuarantees,,1732486488674.cdb3383a4019da5f5ef0568dfacd542e., hostname=bd6592c37bb2,40087,1732486423713, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cdb3383a4019da5f5ef0568dfacd542e, server=bd6592c37bb2,40087,1732486423713 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T22:15:08,785 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732486488674.cdb3383a4019da5f5ef0568dfacd542e., hostname=bd6592c37bb2,40087,1732486423713, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cdb3383a4019da5f5ef0568dfacd542e, server=bd6592c37bb2,40087,1732486423713 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T22:15:08,785 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732486488674.cdb3383a4019da5f5ef0568dfacd542e., hostname=bd6592c37bb2,40087,1732486423713, seqNum=2 because the exception is null or not the one we care about 2024-11-24T22:15:08,786 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/cdb3383a4019da5f5ef0568dfacd542e/.tmp/A/0518f72a50f24932b904aa7048e69b43 as hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/cdb3383a4019da5f5ef0568dfacd542e/A/0518f72a50f24932b904aa7048e69b43 2024-11-24T22:15:08,790 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/cdb3383a4019da5f5ef0568dfacd542e/A/0518f72a50f24932b904aa7048e69b43, entries=150, sequenceid=429, filesize=12.0 K 2024-11-24T22:15:08,791 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/cdb3383a4019da5f5ef0568dfacd542e/.tmp/B/2126a18a8f2f46ba8d5ad912a1e84862 as hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/cdb3383a4019da5f5ef0568dfacd542e/B/2126a18a8f2f46ba8d5ad912a1e84862 2024-11-24T22:15:08,795 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/cdb3383a4019da5f5ef0568dfacd542e/B/2126a18a8f2f46ba8d5ad912a1e84862, entries=150, sequenceid=429, filesize=12.0 K 2024-11-24T22:15:08,795 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/cdb3383a4019da5f5ef0568dfacd542e/.tmp/C/fe3bc641923a4531b63d749f303ae574 as hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/cdb3383a4019da5f5ef0568dfacd542e/C/fe3bc641923a4531b63d749f303ae574 2024-11-24T22:15:08,799 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/cdb3383a4019da5f5ef0568dfacd542e/C/fe3bc641923a4531b63d749f303ae574, entries=150, sequenceid=429, filesize=12.0 K 2024-11-24T22:15:08,800 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~60.38 KB/61830, heapSize ~180 KB/184320, currentSize=114.05 KB/116790 for cdb3383a4019da5f5ef0568dfacd542e in 1257ms, sequenceid=429, compaction requested=true 2024-11-24T22:15:08,800 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for cdb3383a4019da5f5ef0568dfacd542e: 2024-11-24T22:15:08,800 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store cdb3383a4019da5f5ef0568dfacd542e:A, priority=-2147483648, current under compaction store size is 1 2024-11-24T22:15:08,800 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-24T22:15:08,800 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store cdb3383a4019da5f5ef0568dfacd542e:B, priority=-2147483648, current under compaction store size is 2 2024-11-24T22:15:08,800 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-24T22:15:08,800 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store cdb3383a4019da5f5ef0568dfacd542e:C, priority=-2147483648, current under compaction store size is 3 2024-11-24T22:15:08,800 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-24T22:15:08,800 DEBUG [RS:0;bd6592c37bb2:40087-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-24T22:15:08,800 DEBUG [RS:0;bd6592c37bb2:40087-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-24T22:15:08,801 DEBUG [RS:0;bd6592c37bb2:40087-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37789 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-24T22:15:08,801 DEBUG [RS:0;bd6592c37bb2:40087-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37789 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-24T22:15:08,801 DEBUG [RS:0;bd6592c37bb2:40087-shortCompactions-0 {}] regionserver.HStore(1541): cdb3383a4019da5f5ef0568dfacd542e/A is initiating minor compaction (all files) 2024-11-24T22:15:08,801 DEBUG [RS:0;bd6592c37bb2:40087-longCompactions-0 {}] regionserver.HStore(1541): cdb3383a4019da5f5ef0568dfacd542e/B is initiating minor compaction (all files) 2024-11-24T22:15:08,801 INFO [RS:0;bd6592c37bb2:40087-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of cdb3383a4019da5f5ef0568dfacd542e/A in TestAcidGuarantees,,1732486488674.cdb3383a4019da5f5ef0568dfacd542e. 2024-11-24T22:15:08,801 INFO [RS:0;bd6592c37bb2:40087-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of cdb3383a4019da5f5ef0568dfacd542e/B in TestAcidGuarantees,,1732486488674.cdb3383a4019da5f5ef0568dfacd542e. 2024-11-24T22:15:08,801 INFO [RS:0;bd6592c37bb2:40087-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/cdb3383a4019da5f5ef0568dfacd542e/A/110c91754e0a45988a2018e87fa8d232, hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/cdb3383a4019da5f5ef0568dfacd542e/A/ea40fb6886784ae982e1a5b7960a5111, hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/cdb3383a4019da5f5ef0568dfacd542e/A/0518f72a50f24932b904aa7048e69b43] into tmpdir=hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/cdb3383a4019da5f5ef0568dfacd542e/.tmp, totalSize=36.9 K 2024-11-24T22:15:08,801 INFO [RS:0;bd6592c37bb2:40087-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/cdb3383a4019da5f5ef0568dfacd542e/B/e94895b2f12149d9bfc0802c0a2e7d36, hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/cdb3383a4019da5f5ef0568dfacd542e/B/572e0b339dc94be0b36f8ea4a0ef7518, hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/cdb3383a4019da5f5ef0568dfacd542e/B/2126a18a8f2f46ba8d5ad912a1e84862] into tmpdir=hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/cdb3383a4019da5f5ef0568dfacd542e/.tmp, totalSize=36.9 K 2024-11-24T22:15:08,802 DEBUG [RS:0;bd6592c37bb2:40087-longCompactions-0 {}] compactions.Compactor(225): Compacting e94895b2f12149d9bfc0802c0a2e7d36, keycount=150, bloomtype=ROW, size=12.9 K, encoding=NONE, compression=NONE, seqNum=378, earliestPutTs=1732486505960 2024-11-24T22:15:08,802 DEBUG [RS:0;bd6592c37bb2:40087-shortCompactions-0 {}] compactions.Compactor(225): Compacting 110c91754e0a45988a2018e87fa8d232, keycount=150, bloomtype=ROW, size=12.9 K, encoding=NONE, compression=NONE, seqNum=378, earliestPutTs=1732486505960 2024-11-24T22:15:08,802 DEBUG [RS:0;bd6592c37bb2:40087-longCompactions-0 {}] compactions.Compactor(225): Compacting 572e0b339dc94be0b36f8ea4a0ef7518, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=415, earliestPutTs=1732486506791 2024-11-24T22:15:08,802 DEBUG [RS:0;bd6592c37bb2:40087-shortCompactions-0 {}] compactions.Compactor(225): Compacting ea40fb6886784ae982e1a5b7960a5111, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=415, earliestPutTs=1732486506791 2024-11-24T22:15:08,802 DEBUG [RS:0;bd6592c37bb2:40087-longCompactions-0 {}] compactions.Compactor(225): Compacting 2126a18a8f2f46ba8d5ad912a1e84862, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=429, earliestPutTs=1732486507206 2024-11-24T22:15:08,802 DEBUG [RS:0;bd6592c37bb2:40087-shortCompactions-0 {}] compactions.Compactor(225): Compacting 0518f72a50f24932b904aa7048e69b43, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=429, earliestPutTs=1732486507206 2024-11-24T22:15:08,810 INFO [RS:0;bd6592c37bb2:40087-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): cdb3383a4019da5f5ef0568dfacd542e#A#compaction#282 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-24T22:15:08,810 INFO [RS:0;bd6592c37bb2:40087-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): cdb3383a4019da5f5ef0568dfacd542e#B#compaction#283 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-24T22:15:08,810 DEBUG [RS:0;bd6592c37bb2:40087-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/cdb3383a4019da5f5ef0568dfacd542e/.tmp/B/81e4f86e9c7d41eaa6187447800c6c2f is 50, key is test_row_0/B:col10/1732486507215/Put/seqid=0 2024-11-24T22:15:08,810 DEBUG [RS:0;bd6592c37bb2:40087-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/cdb3383a4019da5f5ef0568dfacd542e/.tmp/A/7c55c2e8ae634c89bb30793907b35a80 is 50, key is test_row_0/A:col10/1732486507215/Put/seqid=0 2024-11-24T22:15:08,822 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45929 is added to blk_1073742156_1332 (size=13289) 2024-11-24T22:15:08,839 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45929 is added to blk_1073742157_1333 (size=13289) 2024-11-24T22:15:08,904 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=40087 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=81 2024-11-24T22:15:08,905 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bd6592c37bb2:0-1 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732486488674.cdb3383a4019da5f5ef0568dfacd542e. 2024-11-24T22:15:08,905 INFO [RS_FLUSH_OPERATIONS-regionserver/bd6592c37bb2:0-1 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.HRegion(2902): Flushing cdb3383a4019da5f5ef0568dfacd542e 3/3 column families, dataSize=114.05 KB heapSize=339.42 KB 2024-11-24T22:15:08,905 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bd6592c37bb2:0-1 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.CompactingMemStore(205): FLUSHING TO DISK cdb3383a4019da5f5ef0568dfacd542e, store=A 2024-11-24T22:15:08,905 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bd6592c37bb2:0-1 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-24T22:15:08,905 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bd6592c37bb2:0-1 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.CompactingMemStore(205): FLUSHING TO DISK cdb3383a4019da5f5ef0568dfacd542e, store=B 2024-11-24T22:15:08,905 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bd6592c37bb2:0-1 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-24T22:15:08,905 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bd6592c37bb2:0-1 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.CompactingMemStore(205): FLUSHING TO DISK cdb3383a4019da5f5ef0568dfacd542e, store=C 2024-11-24T22:15:08,905 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bd6592c37bb2:0-1 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-24T22:15:08,910 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bd6592c37bb2:0-1 {event_type=RS_FLUSH_REGIONS, pid=81}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/cdb3383a4019da5f5ef0568dfacd542e/.tmp/A/930ac3ca836c49b0a3f4a7c6e9561a75 is 50, key is test_row_0/A:col10/1732486507604/Put/seqid=0 2024-11-24T22:15:08,914 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45929 is added to blk_1073742158_1334 (size=12301) 2024-11-24T22:15:08,915 INFO [RS_FLUSH_OPERATIONS-regionserver/bd6592c37bb2:0-1 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=38.02 KB at sequenceid=449 (bloomFilter=true), to=hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/cdb3383a4019da5f5ef0568dfacd542e/.tmp/A/930ac3ca836c49b0a3f4a7c6e9561a75 2024-11-24T22:15:08,924 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bd6592c37bb2:0-1 {event_type=RS_FLUSH_REGIONS, pid=81}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/cdb3383a4019da5f5ef0568dfacd542e/.tmp/B/dc4df97aeba4401b8073d9a446f60dd5 is 50, key is test_row_0/B:col10/1732486507604/Put/seqid=0 2024-11-24T22:15:08,928 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45929 is added to blk_1073742159_1335 (size=12301) 2024-11-24T22:15:08,971 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37701 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=80 2024-11-24T22:15:09,227 DEBUG [RS:0;bd6592c37bb2:40087-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/cdb3383a4019da5f5ef0568dfacd542e/.tmp/A/7c55c2e8ae634c89bb30793907b35a80 as hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/cdb3383a4019da5f5ef0568dfacd542e/A/7c55c2e8ae634c89bb30793907b35a80 2024-11-24T22:15:09,233 INFO [RS:0;bd6592c37bb2:40087-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in cdb3383a4019da5f5ef0568dfacd542e/A of cdb3383a4019da5f5ef0568dfacd542e into 7c55c2e8ae634c89bb30793907b35a80(size=13.0 K), total size for store is 13.0 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-24T22:15:09,233 DEBUG [RS:0;bd6592c37bb2:40087-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for cdb3383a4019da5f5ef0568dfacd542e: 2024-11-24T22:15:09,233 INFO [RS:0;bd6592c37bb2:40087-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732486488674.cdb3383a4019da5f5ef0568dfacd542e., storeName=cdb3383a4019da5f5ef0568dfacd542e/A, priority=13, startTime=1732486508800; duration=0sec 2024-11-24T22:15:09,234 DEBUG [RS:0;bd6592c37bb2:40087-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-24T22:15:09,234 DEBUG [RS:0;bd6592c37bb2:40087-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: cdb3383a4019da5f5ef0568dfacd542e:A 2024-11-24T22:15:09,234 DEBUG [RS:0;bd6592c37bb2:40087-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-24T22:15:09,234 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40087 {}] regionserver.HRegion(8855): Flush requested on cdb3383a4019da5f5ef0568dfacd542e 2024-11-24T22:15:09,234 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732486488674.cdb3383a4019da5f5ef0568dfacd542e. as already flushing 2024-11-24T22:15:09,235 DEBUG [RS:0;bd6592c37bb2:40087-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37789 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-24T22:15:09,235 DEBUG [RS:0;bd6592c37bb2:40087-shortCompactions-0 {}] regionserver.HStore(1541): cdb3383a4019da5f5ef0568dfacd542e/C is initiating minor compaction (all files) 2024-11-24T22:15:09,235 INFO [RS:0;bd6592c37bb2:40087-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of cdb3383a4019da5f5ef0568dfacd542e/C in TestAcidGuarantees,,1732486488674.cdb3383a4019da5f5ef0568dfacd542e. 2024-11-24T22:15:09,235 INFO [RS:0;bd6592c37bb2:40087-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/cdb3383a4019da5f5ef0568dfacd542e/C/f4856e4fded1464d906bb6d736463799, hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/cdb3383a4019da5f5ef0568dfacd542e/C/845e21e625844395ad910b7963d0e3fc, hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/cdb3383a4019da5f5ef0568dfacd542e/C/fe3bc641923a4531b63d749f303ae574] into tmpdir=hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/cdb3383a4019da5f5ef0568dfacd542e/.tmp, totalSize=36.9 K 2024-11-24T22:15:09,236 DEBUG [RS:0;bd6592c37bb2:40087-shortCompactions-0 {}] compactions.Compactor(225): Compacting f4856e4fded1464d906bb6d736463799, keycount=150, bloomtype=ROW, size=12.9 K, encoding=NONE, compression=NONE, seqNum=378, earliestPutTs=1732486505960 2024-11-24T22:15:09,236 DEBUG [RS:0;bd6592c37bb2:40087-shortCompactions-0 {}] compactions.Compactor(225): Compacting 845e21e625844395ad910b7963d0e3fc, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=415, earliestPutTs=1732486506791 2024-11-24T22:15:09,236 DEBUG [RS:0;bd6592c37bb2:40087-shortCompactions-0 {}] compactions.Compactor(225): Compacting fe3bc641923a4531b63d749f303ae574, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=429, earliestPutTs=1732486507206 2024-11-24T22:15:09,248 DEBUG [RS:0;bd6592c37bb2:40087-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/cdb3383a4019da5f5ef0568dfacd542e/.tmp/B/81e4f86e9c7d41eaa6187447800c6c2f as hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/cdb3383a4019da5f5ef0568dfacd542e/B/81e4f86e9c7d41eaa6187447800c6c2f 2024-11-24T22:15:09,251 INFO [RS:0;bd6592c37bb2:40087-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): cdb3383a4019da5f5ef0568dfacd542e#C#compaction#286 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-24T22:15:09,251 DEBUG [RS:0;bd6592c37bb2:40087-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/cdb3383a4019da5f5ef0568dfacd542e/.tmp/C/e522bb98c69848cf83b98d352440eb70 is 50, key is test_row_0/C:col10/1732486507215/Put/seqid=0 2024-11-24T22:15:09,258 INFO [RS:0;bd6592c37bb2:40087-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in cdb3383a4019da5f5ef0568dfacd542e/B of cdb3383a4019da5f5ef0568dfacd542e into 81e4f86e9c7d41eaa6187447800c6c2f(size=13.0 K), total size for store is 13.0 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-24T22:15:09,259 DEBUG [RS:0;bd6592c37bb2:40087-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for cdb3383a4019da5f5ef0568dfacd542e: 2024-11-24T22:15:09,259 INFO [RS:0;bd6592c37bb2:40087-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732486488674.cdb3383a4019da5f5ef0568dfacd542e., storeName=cdb3383a4019da5f5ef0568dfacd542e/B, priority=13, startTime=1732486508800; duration=0sec 2024-11-24T22:15:09,259 DEBUG [RS:0;bd6592c37bb2:40087-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-24T22:15:09,259 DEBUG [RS:0;bd6592c37bb2:40087-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: cdb3383a4019da5f5ef0568dfacd542e:B 2024-11-24T22:15:09,260 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45929 is added to blk_1073742160_1336 (size=13289) 2024-11-24T22:15:09,279 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40087 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cdb3383a4019da5f5ef0568dfacd542e, server=bd6592c37bb2,40087,1732486423713 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-24T22:15:09,279 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40087 {}] ipc.CallRunner(138): callId: 22 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40772 deadline: 1732486569275, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cdb3383a4019da5f5ef0568dfacd542e, server=bd6592c37bb2,40087,1732486423713 2024-11-24T22:15:09,280 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732486488674.cdb3383a4019da5f5ef0568dfacd542e., hostname=bd6592c37bb2,40087,1732486423713, seqNum=2 , the old value is region=TestAcidGuarantees,,1732486488674.cdb3383a4019da5f5ef0568dfacd542e., hostname=bd6592c37bb2,40087,1732486423713, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cdb3383a4019da5f5ef0568dfacd542e, server=bd6592c37bb2,40087,1732486423713 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T22:15:09,280 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732486488674.cdb3383a4019da5f5ef0568dfacd542e., hostname=bd6592c37bb2,40087,1732486423713, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cdb3383a4019da5f5ef0568dfacd542e, server=bd6592c37bb2,40087,1732486423713 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T22:15:09,280 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732486488674.cdb3383a4019da5f5ef0568dfacd542e., hostname=bd6592c37bb2,40087,1732486423713, seqNum=2 because the exception is null or not the one we care about 2024-11-24T22:15:09,284 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40087 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cdb3383a4019da5f5ef0568dfacd542e, server=bd6592c37bb2,40087,1732486423713 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-24T22:15:09,284 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40087 {}] ipc.CallRunner(138): callId: 19 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40842 deadline: 1732486569278, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cdb3383a4019da5f5ef0568dfacd542e, server=bd6592c37bb2,40087,1732486423713 2024-11-24T22:15:09,284 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40087 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cdb3383a4019da5f5ef0568dfacd542e, server=bd6592c37bb2,40087,1732486423713 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-24T22:15:09,284 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732486488674.cdb3383a4019da5f5ef0568dfacd542e., hostname=bd6592c37bb2,40087,1732486423713, seqNum=2 , the old value is region=TestAcidGuarantees,,1732486488674.cdb3383a4019da5f5ef0568dfacd542e., hostname=bd6592c37bb2,40087,1732486423713, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cdb3383a4019da5f5ef0568dfacd542e, server=bd6592c37bb2,40087,1732486423713 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T22:15:09,285 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40087 {}] ipc.CallRunner(138): callId: 18 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40816 deadline: 1732486569282, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cdb3383a4019da5f5ef0568dfacd542e, server=bd6592c37bb2,40087,1732486423713 2024-11-24T22:15:09,285 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732486488674.cdb3383a4019da5f5ef0568dfacd542e., hostname=bd6592c37bb2,40087,1732486423713, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cdb3383a4019da5f5ef0568dfacd542e, server=bd6592c37bb2,40087,1732486423713 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T22:15:09,285 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732486488674.cdb3383a4019da5f5ef0568dfacd542e., hostname=bd6592c37bb2,40087,1732486423713, seqNum=2 because the exception is null or not the one we care about 2024-11-24T22:15:09,285 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40087 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cdb3383a4019da5f5ef0568dfacd542e, server=bd6592c37bb2,40087,1732486423713 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-24T22:15:09,285 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40087 {}] ipc.CallRunner(138): callId: 23 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40806 deadline: 1732486569282, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cdb3383a4019da5f5ef0568dfacd542e, server=bd6592c37bb2,40087,1732486423713 2024-11-24T22:15:09,285 WARN [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRpcRetryingCaller(177): Call to bd6592c37bb2,40087,1732486423713 for 'test_row_2' in cdb3383a4019da5f5ef0568dfacd542e of TestAcidGuarantees failed, tries = 9, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 18327 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cdb3383a4019da5f5ef0568dfacd542e, server=bd6592c37bb2,40087,1732486423713 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor44.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cdb3383a4019da5f5ef0568dfacd542e, server=bd6592c37bb2,40087,1732486423713 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-24T22:15:09,286 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732486488674.cdb3383a4019da5f5ef0568dfacd542e., hostname=bd6592c37bb2,40087,1732486423713, seqNum=2 , the old value is region=TestAcidGuarantees,,1732486488674.cdb3383a4019da5f5ef0568dfacd542e., hostname=bd6592c37bb2,40087,1732486423713, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cdb3383a4019da5f5ef0568dfacd542e, server=bd6592c37bb2,40087,1732486423713 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T22:15:09,286 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732486488674.cdb3383a4019da5f5ef0568dfacd542e., hostname=bd6592c37bb2,40087,1732486423713, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cdb3383a4019da5f5ef0568dfacd542e, server=bd6592c37bb2,40087,1732486423713 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T22:15:09,286 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732486488674.cdb3383a4019da5f5ef0568dfacd542e., hostname=bd6592c37bb2,40087,1732486423713, seqNum=2 because the exception is null or not the one we care about 2024-11-24T22:15:09,286 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732486488674.cdb3383a4019da5f5ef0568dfacd542e., hostname=bd6592c37bb2,40087,1732486423713, seqNum=2 , the old value is region=TestAcidGuarantees,,1732486488674.cdb3383a4019da5f5ef0568dfacd542e., hostname=bd6592c37bb2,40087,1732486423713, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cdb3383a4019da5f5ef0568dfacd542e, server=bd6592c37bb2,40087,1732486423713 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T22:15:09,286 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732486488674.cdb3383a4019da5f5ef0568dfacd542e., hostname=bd6592c37bb2,40087,1732486423713, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cdb3383a4019da5f5ef0568dfacd542e, server=bd6592c37bb2,40087,1732486423713 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T22:15:09,286 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732486488674.cdb3383a4019da5f5ef0568dfacd542e., hostname=bd6592c37bb2,40087,1732486423713, seqNum=2 because the exception is null or not the one we care about 2024-11-24T22:15:09,329 INFO [RS_FLUSH_OPERATIONS-regionserver/bd6592c37bb2:0-1 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=38.02 KB at sequenceid=449 (bloomFilter=true), to=hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/cdb3383a4019da5f5ef0568dfacd542e/.tmp/B/dc4df97aeba4401b8073d9a446f60dd5 2024-11-24T22:15:09,338 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bd6592c37bb2:0-1 {event_type=RS_FLUSH_REGIONS, pid=81}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/cdb3383a4019da5f5ef0568dfacd542e/.tmp/C/2bb428b726714735a7a3359a0464ba12 is 50, key is test_row_0/C:col10/1732486507604/Put/seqid=0 2024-11-24T22:15:09,341 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45929 is added to blk_1073742161_1337 (size=12301) 2024-11-24T22:15:09,392 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40087 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cdb3383a4019da5f5ef0568dfacd542e, server=bd6592c37bb2,40087,1732486423713 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-24T22:15:09,393 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40087 {}] ipc.CallRunner(138): callId: 23 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40772 deadline: 1732486569391, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cdb3383a4019da5f5ef0568dfacd542e, server=bd6592c37bb2,40087,1732486423713 2024-11-24T22:15:09,393 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40087 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cdb3383a4019da5f5ef0568dfacd542e, server=bd6592c37bb2,40087,1732486423713 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-24T22:15:09,393 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732486488674.cdb3383a4019da5f5ef0568dfacd542e., hostname=bd6592c37bb2,40087,1732486423713, seqNum=2 , the old value is region=TestAcidGuarantees,,1732486488674.cdb3383a4019da5f5ef0568dfacd542e., hostname=bd6592c37bb2,40087,1732486423713, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cdb3383a4019da5f5ef0568dfacd542e, server=bd6592c37bb2,40087,1732486423713 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T22:15:09,393 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732486488674.cdb3383a4019da5f5ef0568dfacd542e., hostname=bd6592c37bb2,40087,1732486423713, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cdb3383a4019da5f5ef0568dfacd542e, server=bd6592c37bb2,40087,1732486423713 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T22:15:09,393 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40087 {}] ipc.CallRunner(138): callId: 20 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40842 deadline: 1732486569391, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cdb3383a4019da5f5ef0568dfacd542e, server=bd6592c37bb2,40087,1732486423713 2024-11-24T22:15:09,393 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732486488674.cdb3383a4019da5f5ef0568dfacd542e., hostname=bd6592c37bb2,40087,1732486423713, seqNum=2 because the exception is null or not the one we care about 2024-11-24T22:15:09,393 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40087 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cdb3383a4019da5f5ef0568dfacd542e, server=bd6592c37bb2,40087,1732486423713 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-24T22:15:09,393 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732486488674.cdb3383a4019da5f5ef0568dfacd542e., hostname=bd6592c37bb2,40087,1732486423713, seqNum=2 , the old value is region=TestAcidGuarantees,,1732486488674.cdb3383a4019da5f5ef0568dfacd542e., hostname=bd6592c37bb2,40087,1732486423713, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cdb3383a4019da5f5ef0568dfacd542e, server=bd6592c37bb2,40087,1732486423713 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T22:15:09,393 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40087 {}] ipc.CallRunner(138): callId: 24 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40806 deadline: 1732486569391, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cdb3383a4019da5f5ef0568dfacd542e, server=bd6592c37bb2,40087,1732486423713 2024-11-24T22:15:09,393 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732486488674.cdb3383a4019da5f5ef0568dfacd542e., hostname=bd6592c37bb2,40087,1732486423713, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cdb3383a4019da5f5ef0568dfacd542e, server=bd6592c37bb2,40087,1732486423713 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T22:15:09,393 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732486488674.cdb3383a4019da5f5ef0568dfacd542e., hostname=bd6592c37bb2,40087,1732486423713, seqNum=2 because the exception is null or not the one we care about 2024-11-24T22:15:09,394 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732486488674.cdb3383a4019da5f5ef0568dfacd542e., hostname=bd6592c37bb2,40087,1732486423713, seqNum=2 , the old value is region=TestAcidGuarantees,,1732486488674.cdb3383a4019da5f5ef0568dfacd542e., hostname=bd6592c37bb2,40087,1732486423713, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cdb3383a4019da5f5ef0568dfacd542e, server=bd6592c37bb2,40087,1732486423713 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T22:15:09,394 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732486488674.cdb3383a4019da5f5ef0568dfacd542e., hostname=bd6592c37bb2,40087,1732486423713, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cdb3383a4019da5f5ef0568dfacd542e, server=bd6592c37bb2,40087,1732486423713 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T22:15:09,394 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732486488674.cdb3383a4019da5f5ef0568dfacd542e., hostname=bd6592c37bb2,40087,1732486423713, seqNum=2 because the exception is null or not the one we care about 2024-11-24T22:15:09,601 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40087 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cdb3383a4019da5f5ef0568dfacd542e, server=bd6592c37bb2,40087,1732486423713 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-24T22:15:09,601 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40087 {}] ipc.CallRunner(138): callId: 24 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40772 deadline: 1732486569600, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cdb3383a4019da5f5ef0568dfacd542e, server=bd6592c37bb2,40087,1732486423713 2024-11-24T22:15:09,602 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40087 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cdb3383a4019da5f5ef0568dfacd542e, server=bd6592c37bb2,40087,1732486423713 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-24T22:15:09,602 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732486488674.cdb3383a4019da5f5ef0568dfacd542e., hostname=bd6592c37bb2,40087,1732486423713, seqNum=2 , the old value is region=TestAcidGuarantees,,1732486488674.cdb3383a4019da5f5ef0568dfacd542e., hostname=bd6592c37bb2,40087,1732486423713, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cdb3383a4019da5f5ef0568dfacd542e, server=bd6592c37bb2,40087,1732486423713 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T22:15:09,602 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40087 {}] ipc.CallRunner(138): callId: 21 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40842 deadline: 1732486569601, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cdb3383a4019da5f5ef0568dfacd542e, server=bd6592c37bb2,40087,1732486423713 2024-11-24T22:15:09,602 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732486488674.cdb3383a4019da5f5ef0568dfacd542e., hostname=bd6592c37bb2,40087,1732486423713, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cdb3383a4019da5f5ef0568dfacd542e, server=bd6592c37bb2,40087,1732486423713 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T22:15:09,602 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732486488674.cdb3383a4019da5f5ef0568dfacd542e., hostname=bd6592c37bb2,40087,1732486423713, seqNum=2 because the exception is null or not the one we care about 2024-11-24T22:15:09,602 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40087 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cdb3383a4019da5f5ef0568dfacd542e, server=bd6592c37bb2,40087,1732486423713 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-24T22:15:09,602 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732486488674.cdb3383a4019da5f5ef0568dfacd542e., hostname=bd6592c37bb2,40087,1732486423713, seqNum=2 , the old value is region=TestAcidGuarantees,,1732486488674.cdb3383a4019da5f5ef0568dfacd542e., hostname=bd6592c37bb2,40087,1732486423713, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cdb3383a4019da5f5ef0568dfacd542e, server=bd6592c37bb2,40087,1732486423713 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T22:15:09,602 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732486488674.cdb3383a4019da5f5ef0568dfacd542e., hostname=bd6592c37bb2,40087,1732486423713, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cdb3383a4019da5f5ef0568dfacd542e, server=bd6592c37bb2,40087,1732486423713 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T22:15:09,602 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40087 {}] ipc.CallRunner(138): callId: 25 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40806 deadline: 1732486569601, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cdb3383a4019da5f5ef0568dfacd542e, server=bd6592c37bb2,40087,1732486423713 2024-11-24T22:15:09,602 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732486488674.cdb3383a4019da5f5ef0568dfacd542e., hostname=bd6592c37bb2,40087,1732486423713, seqNum=2 because the exception is null or not the one we care about 2024-11-24T22:15:09,602 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732486488674.cdb3383a4019da5f5ef0568dfacd542e., hostname=bd6592c37bb2,40087,1732486423713, seqNum=2 , the old value is region=TestAcidGuarantees,,1732486488674.cdb3383a4019da5f5ef0568dfacd542e., hostname=bd6592c37bb2,40087,1732486423713, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cdb3383a4019da5f5ef0568dfacd542e, server=bd6592c37bb2,40087,1732486423713 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T22:15:09,602 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732486488674.cdb3383a4019da5f5ef0568dfacd542e., hostname=bd6592c37bb2,40087,1732486423713, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cdb3383a4019da5f5ef0568dfacd542e, server=bd6592c37bb2,40087,1732486423713 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T22:15:09,602 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732486488674.cdb3383a4019da5f5ef0568dfacd542e., hostname=bd6592c37bb2,40087,1732486423713, seqNum=2 because the exception is null or not the one we care about 2024-11-24T22:15:09,665 DEBUG [RS:0;bd6592c37bb2:40087-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/cdb3383a4019da5f5ef0568dfacd542e/.tmp/C/e522bb98c69848cf83b98d352440eb70 as hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/cdb3383a4019da5f5ef0568dfacd542e/C/e522bb98c69848cf83b98d352440eb70 2024-11-24T22:15:09,670 INFO [RS:0;bd6592c37bb2:40087-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in cdb3383a4019da5f5ef0568dfacd542e/C of cdb3383a4019da5f5ef0568dfacd542e into e522bb98c69848cf83b98d352440eb70(size=13.0 K), total size for store is 13.0 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-24T22:15:09,670 DEBUG [RS:0;bd6592c37bb2:40087-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for cdb3383a4019da5f5ef0568dfacd542e: 2024-11-24T22:15:09,670 INFO [RS:0;bd6592c37bb2:40087-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732486488674.cdb3383a4019da5f5ef0568dfacd542e., storeName=cdb3383a4019da5f5ef0568dfacd542e/C, priority=13, startTime=1732486508800; duration=0sec 2024-11-24T22:15:09,670 DEBUG [RS:0;bd6592c37bb2:40087-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-24T22:15:09,671 DEBUG [RS:0;bd6592c37bb2:40087-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: cdb3383a4019da5f5ef0568dfacd542e:C 2024-11-24T22:15:09,742 INFO [RS_FLUSH_OPERATIONS-regionserver/bd6592c37bb2:0-1 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=38.02 KB at sequenceid=449 (bloomFilter=true), to=hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/cdb3383a4019da5f5ef0568dfacd542e/.tmp/C/2bb428b726714735a7a3359a0464ba12 2024-11-24T22:15:09,747 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bd6592c37bb2:0-1 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/cdb3383a4019da5f5ef0568dfacd542e/.tmp/A/930ac3ca836c49b0a3f4a7c6e9561a75 as hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/cdb3383a4019da5f5ef0568dfacd542e/A/930ac3ca836c49b0a3f4a7c6e9561a75 2024-11-24T22:15:09,751 INFO [RS_FLUSH_OPERATIONS-regionserver/bd6592c37bb2:0-1 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/cdb3383a4019da5f5ef0568dfacd542e/A/930ac3ca836c49b0a3f4a7c6e9561a75, entries=150, sequenceid=449, filesize=12.0 K 2024-11-24T22:15:09,752 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bd6592c37bb2:0-1 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/cdb3383a4019da5f5ef0568dfacd542e/.tmp/B/dc4df97aeba4401b8073d9a446f60dd5 as hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/cdb3383a4019da5f5ef0568dfacd542e/B/dc4df97aeba4401b8073d9a446f60dd5 2024-11-24T22:15:09,755 INFO [RS_FLUSH_OPERATIONS-regionserver/bd6592c37bb2:0-1 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/cdb3383a4019da5f5ef0568dfacd542e/B/dc4df97aeba4401b8073d9a446f60dd5, entries=150, sequenceid=449, filesize=12.0 K 2024-11-24T22:15:09,756 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bd6592c37bb2:0-1 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/cdb3383a4019da5f5ef0568dfacd542e/.tmp/C/2bb428b726714735a7a3359a0464ba12 as hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/cdb3383a4019da5f5ef0568dfacd542e/C/2bb428b726714735a7a3359a0464ba12 2024-11-24T22:15:09,759 INFO [RS_FLUSH_OPERATIONS-regionserver/bd6592c37bb2:0-1 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/cdb3383a4019da5f5ef0568dfacd542e/C/2bb428b726714735a7a3359a0464ba12, entries=150, sequenceid=449, filesize=12.0 K 2024-11-24T22:15:09,760 INFO [RS_FLUSH_OPERATIONS-regionserver/bd6592c37bb2:0-1 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.HRegion(3140): Finished flush of dataSize ~114.05 KB/116790, heapSize ~339.38 KB/347520, currentSize=60.38 KB/61830 for cdb3383a4019da5f5ef0568dfacd542e in 855ms, sequenceid=449, compaction requested=false 2024-11-24T22:15:09,760 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bd6592c37bb2:0-1 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.HRegion(2603): Flush status journal for cdb3383a4019da5f5ef0568dfacd542e: 2024-11-24T22:15:09,760 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bd6592c37bb2:0-1 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732486488674.cdb3383a4019da5f5ef0568dfacd542e. 2024-11-24T22:15:09,760 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bd6592c37bb2:0-1 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=81 2024-11-24T22:15:09,761 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37701 {}] master.HMaster(4169): Remote procedure done, pid=81 2024-11-24T22:15:09,765 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=81, resume processing ppid=80 2024-11-24T22:15:09,765 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=81, ppid=80, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.9270 sec 2024-11-24T22:15:09,767 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=80, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=80, table=TestAcidGuarantees in 1.9330 sec 2024-11-24T22:15:09,804 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40087 {}] regionserver.HRegion(8855): Flush requested on cdb3383a4019da5f5ef0568dfacd542e 2024-11-24T22:15:09,804 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing cdb3383a4019da5f5ef0568dfacd542e 3/3 column families, dataSize=67.09 KB heapSize=199.97 KB 2024-11-24T22:15:09,804 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK cdb3383a4019da5f5ef0568dfacd542e, store=A 2024-11-24T22:15:09,804 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-24T22:15:09,804 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK cdb3383a4019da5f5ef0568dfacd542e, store=B 2024-11-24T22:15:09,804 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-24T22:15:09,804 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK cdb3383a4019da5f5ef0568dfacd542e, store=C 2024-11-24T22:15:09,804 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-24T22:15:09,809 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/cdb3383a4019da5f5ef0568dfacd542e/.tmp/A/c60630f768c245218991d2174c071e39 is 50, key is test_row_0/A:col10/1732486509274/Put/seqid=0 2024-11-24T22:15:09,818 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45929 is added to blk_1073742162_1338 (size=14741) 2024-11-24T22:15:09,821 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=465 (bloomFilter=true), to=hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/cdb3383a4019da5f5ef0568dfacd542e/.tmp/A/c60630f768c245218991d2174c071e39 2024-11-24T22:15:09,830 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/cdb3383a4019da5f5ef0568dfacd542e/.tmp/B/4e906e5b872246cba035629dd76e68f0 is 50, key is test_row_0/B:col10/1732486509274/Put/seqid=0 2024-11-24T22:15:09,842 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45929 is added to blk_1073742163_1339 (size=12301) 2024-11-24T22:15:09,894 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40087 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cdb3383a4019da5f5ef0568dfacd542e, server=bd6592c37bb2,40087,1732486423713 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-24T22:15:09,895 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40087 {}] ipc.CallRunner(138): callId: 398 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40834 deadline: 1732486569889, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cdb3383a4019da5f5ef0568dfacd542e, server=bd6592c37bb2,40087,1732486423713 2024-11-24T22:15:09,895 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732486488674.cdb3383a4019da5f5ef0568dfacd542e., hostname=bd6592c37bb2,40087,1732486423713, seqNum=2 , the old value is region=TestAcidGuarantees,,1732486488674.cdb3383a4019da5f5ef0568dfacd542e., hostname=bd6592c37bb2,40087,1732486423713, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cdb3383a4019da5f5ef0568dfacd542e, server=bd6592c37bb2,40087,1732486423713 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T22:15:09,895 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732486488674.cdb3383a4019da5f5ef0568dfacd542e., hostname=bd6592c37bb2,40087,1732486423713, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cdb3383a4019da5f5ef0568dfacd542e, server=bd6592c37bb2,40087,1732486423713 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T22:15:09,895 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732486488674.cdb3383a4019da5f5ef0568dfacd542e., hostname=bd6592c37bb2,40087,1732486423713, seqNum=2 because the exception is null or not the one we care about 2024-11-24T22:15:09,914 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40087 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cdb3383a4019da5f5ef0568dfacd542e, server=bd6592c37bb2,40087,1732486423713 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-24T22:15:09,914 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40087 {}] ipc.CallRunner(138): callId: 25 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40772 deadline: 1732486569910, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cdb3383a4019da5f5ef0568dfacd542e, server=bd6592c37bb2,40087,1732486423713 2024-11-24T22:15:09,914 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40087 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cdb3383a4019da5f5ef0568dfacd542e, server=bd6592c37bb2,40087,1732486423713 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-24T22:15:09,914 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732486488674.cdb3383a4019da5f5ef0568dfacd542e., hostname=bd6592c37bb2,40087,1732486423713, seqNum=2 , the old value is region=TestAcidGuarantees,,1732486488674.cdb3383a4019da5f5ef0568dfacd542e., hostname=bd6592c37bb2,40087,1732486423713, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cdb3383a4019da5f5ef0568dfacd542e, server=bd6592c37bb2,40087,1732486423713 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T22:15:09,914 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732486488674.cdb3383a4019da5f5ef0568dfacd542e., hostname=bd6592c37bb2,40087,1732486423713, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cdb3383a4019da5f5ef0568dfacd542e, server=bd6592c37bb2,40087,1732486423713 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T22:15:09,914 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40087 {}] ipc.CallRunner(138): callId: 26 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40806 deadline: 1732486569910, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cdb3383a4019da5f5ef0568dfacd542e, server=bd6592c37bb2,40087,1732486423713 2024-11-24T22:15:09,914 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732486488674.cdb3383a4019da5f5ef0568dfacd542e., hostname=bd6592c37bb2,40087,1732486423713, seqNum=2 because the exception is null or not the one we care about 2024-11-24T22:15:09,915 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732486488674.cdb3383a4019da5f5ef0568dfacd542e., hostname=bd6592c37bb2,40087,1732486423713, seqNum=2 , the old value is region=TestAcidGuarantees,,1732486488674.cdb3383a4019da5f5ef0568dfacd542e., hostname=bd6592c37bb2,40087,1732486423713, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cdb3383a4019da5f5ef0568dfacd542e, server=bd6592c37bb2,40087,1732486423713 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T22:15:09,915 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40087 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cdb3383a4019da5f5ef0568dfacd542e, server=bd6592c37bb2,40087,1732486423713 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-24T22:15:09,915 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732486488674.cdb3383a4019da5f5ef0568dfacd542e., hostname=bd6592c37bb2,40087,1732486423713, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cdb3383a4019da5f5ef0568dfacd542e, server=bd6592c37bb2,40087,1732486423713 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T22:15:09,915 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732486488674.cdb3383a4019da5f5ef0568dfacd542e., hostname=bd6592c37bb2,40087,1732486423713, seqNum=2 because the exception is null or not the one we care about 2024-11-24T22:15:09,915 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40087 {}] ipc.CallRunner(138): callId: 22 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40842 deadline: 1732486569911, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cdb3383a4019da5f5ef0568dfacd542e, server=bd6592c37bb2,40087,1732486423713 2024-11-24T22:15:09,915 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732486488674.cdb3383a4019da5f5ef0568dfacd542e., hostname=bd6592c37bb2,40087,1732486423713, seqNum=2 , the old value is region=TestAcidGuarantees,,1732486488674.cdb3383a4019da5f5ef0568dfacd542e., hostname=bd6592c37bb2,40087,1732486423713, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cdb3383a4019da5f5ef0568dfacd542e, server=bd6592c37bb2,40087,1732486423713 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T22:15:09,915 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732486488674.cdb3383a4019da5f5ef0568dfacd542e., hostname=bd6592c37bb2,40087,1732486423713, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cdb3383a4019da5f5ef0568dfacd542e, server=bd6592c37bb2,40087,1732486423713 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T22:15:09,915 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732486488674.cdb3383a4019da5f5ef0568dfacd542e., hostname=bd6592c37bb2,40087,1732486423713, seqNum=2 because the exception is null or not the one we care about 2024-11-24T22:15:09,981 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37701 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=80 2024-11-24T22:15:09,981 INFO [RPCClient-NioEventLoopGroup-4-6 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-24T22:15:09,982 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37701 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-24T22:15:09,983 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37701 {}] procedure2.ProcedureExecutor(1139): Stored pid=82, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=82, table=TestAcidGuarantees 2024-11-24T22:15:09,983 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37701 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=82 2024-11-24T22:15:09,984 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=82, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=82, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-24T22:15:09,984 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=82, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=82, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-24T22:15:09,985 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=83, ppid=82, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-24T22:15:10,004 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40087 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cdb3383a4019da5f5ef0568dfacd542e, server=bd6592c37bb2,40087,1732486423713 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-24T22:15:10,005 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40087 {}] ipc.CallRunner(138): callId: 399 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40834 deadline: 1732486570000, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cdb3383a4019da5f5ef0568dfacd542e, server=bd6592c37bb2,40087,1732486423713 2024-11-24T22:15:10,005 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732486488674.cdb3383a4019da5f5ef0568dfacd542e., hostname=bd6592c37bb2,40087,1732486423713, seqNum=2 , the old value is region=TestAcidGuarantees,,1732486488674.cdb3383a4019da5f5ef0568dfacd542e., hostname=bd6592c37bb2,40087,1732486423713, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cdb3383a4019da5f5ef0568dfacd542e, server=bd6592c37bb2,40087,1732486423713 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T22:15:10,005 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732486488674.cdb3383a4019da5f5ef0568dfacd542e., hostname=bd6592c37bb2,40087,1732486423713, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cdb3383a4019da5f5ef0568dfacd542e, server=bd6592c37bb2,40087,1732486423713 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T22:15:10,005 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732486488674.cdb3383a4019da5f5ef0568dfacd542e., hostname=bd6592c37bb2,40087,1732486423713, seqNum=2 because the exception is null or not the one we care about 2024-11-24T22:15:10,091 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37701 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=82 2024-11-24T22:15:10,137 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=40087 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=83 2024-11-24T22:15:10,137 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bd6592c37bb2:0-2 {event_type=RS_FLUSH_REGIONS, pid=83}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732486488674.cdb3383a4019da5f5ef0568dfacd542e. 2024-11-24T22:15:10,137 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bd6592c37bb2:0-2 {event_type=RS_FLUSH_REGIONS, pid=83}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732486488674.cdb3383a4019da5f5ef0568dfacd542e. as already flushing 2024-11-24T22:15:10,137 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bd6592c37bb2:0-2 {event_type=RS_FLUSH_REGIONS, pid=83}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732486488674.cdb3383a4019da5f5ef0568dfacd542e. 2024-11-24T22:15:10,137 ERROR [RS_FLUSH_OPERATIONS-regionserver/bd6592c37bb2:0-2 {event_type=RS_FLUSH_REGIONS, pid=83}] handler.RSProcedureHandler(58): pid=83 java.io.IOException: Unable to complete flush {ENCODED => cdb3383a4019da5f5ef0568dfacd542e, NAME => 'TestAcidGuarantees,,1732486488674.cdb3383a4019da5f5ef0568dfacd542e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T22:15:10,137 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bd6592c37bb2:0-2 {event_type=RS_FLUSH_REGIONS, pid=83}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=83 java.io.IOException: Unable to complete flush {ENCODED => cdb3383a4019da5f5ef0568dfacd542e, NAME => 'TestAcidGuarantees,,1732486488674.cdb3383a4019da5f5ef0568dfacd542e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T22:15:10,138 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37701 {}] master.HMaster(4177): Remote procedure failed, pid=83 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => cdb3383a4019da5f5ef0568dfacd542e, NAME => 'TestAcidGuarantees,,1732486488674.cdb3383a4019da5f5ef0568dfacd542e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => cdb3383a4019da5f5ef0568dfacd542e, NAME => 'TestAcidGuarantees,,1732486488674.cdb3383a4019da5f5ef0568dfacd542e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T22:15:10,216 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40087 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cdb3383a4019da5f5ef0568dfacd542e, server=bd6592c37bb2,40087,1732486423713 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-24T22:15:10,216 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40087 {}] ipc.CallRunner(138): callId: 400 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40834 deadline: 1732486570211, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cdb3383a4019da5f5ef0568dfacd542e, server=bd6592c37bb2,40087,1732486423713 2024-11-24T22:15:10,216 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732486488674.cdb3383a4019da5f5ef0568dfacd542e., hostname=bd6592c37bb2,40087,1732486423713, seqNum=2 , the old value is region=TestAcidGuarantees,,1732486488674.cdb3383a4019da5f5ef0568dfacd542e., hostname=bd6592c37bb2,40087,1732486423713, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cdb3383a4019da5f5ef0568dfacd542e, server=bd6592c37bb2,40087,1732486423713 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T22:15:10,216 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732486488674.cdb3383a4019da5f5ef0568dfacd542e., hostname=bd6592c37bb2,40087,1732486423713, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cdb3383a4019da5f5ef0568dfacd542e, server=bd6592c37bb2,40087,1732486423713 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T22:15:10,216 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732486488674.cdb3383a4019da5f5ef0568dfacd542e., hostname=bd6592c37bb2,40087,1732486423713, seqNum=2 because the exception is null or not the one we care about 2024-11-24T22:15:10,243 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=465 (bloomFilter=true), to=hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/cdb3383a4019da5f5ef0568dfacd542e/.tmp/B/4e906e5b872246cba035629dd76e68f0 2024-11-24T22:15:10,253 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/cdb3383a4019da5f5ef0568dfacd542e/.tmp/C/9dcfc731809446bdbe1c35518b60caee is 50, key is test_row_0/C:col10/1732486509274/Put/seqid=0 2024-11-24T22:15:10,262 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45929 is added to blk_1073742164_1340 (size=12301) 2024-11-24T22:15:10,290 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=40087 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=83 2024-11-24T22:15:10,290 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bd6592c37bb2:0-0 {event_type=RS_FLUSH_REGIONS, pid=83}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732486488674.cdb3383a4019da5f5ef0568dfacd542e. 2024-11-24T22:15:10,290 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bd6592c37bb2:0-0 {event_type=RS_FLUSH_REGIONS, pid=83}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732486488674.cdb3383a4019da5f5ef0568dfacd542e. as already flushing 2024-11-24T22:15:10,290 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bd6592c37bb2:0-0 {event_type=RS_FLUSH_REGIONS, pid=83}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732486488674.cdb3383a4019da5f5ef0568dfacd542e. 2024-11-24T22:15:10,290 ERROR [RS_FLUSH_OPERATIONS-regionserver/bd6592c37bb2:0-0 {event_type=RS_FLUSH_REGIONS, pid=83}] handler.RSProcedureHandler(58): pid=83 java.io.IOException: Unable to complete flush {ENCODED => cdb3383a4019da5f5ef0568dfacd542e, NAME => 'TestAcidGuarantees,,1732486488674.cdb3383a4019da5f5ef0568dfacd542e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T22:15:10,290 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bd6592c37bb2:0-0 {event_type=RS_FLUSH_REGIONS, pid=83}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=83 java.io.IOException: Unable to complete flush {ENCODED => cdb3383a4019da5f5ef0568dfacd542e, NAME => 'TestAcidGuarantees,,1732486488674.cdb3383a4019da5f5ef0568dfacd542e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T22:15:10,291 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37701 {}] master.HMaster(4177): Remote procedure failed, pid=83 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => cdb3383a4019da5f5ef0568dfacd542e, NAME => 'TestAcidGuarantees,,1732486488674.cdb3383a4019da5f5ef0568dfacd542e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => cdb3383a4019da5f5ef0568dfacd542e, NAME => 'TestAcidGuarantees,,1732486488674.cdb3383a4019da5f5ef0568dfacd542e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T22:15:10,301 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37701 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=82 2024-11-24T22:15:10,425 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40087 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cdb3383a4019da5f5ef0568dfacd542e, server=bd6592c37bb2,40087,1732486423713 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-24T22:15:10,425 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40087 {}] ipc.CallRunner(138): callId: 26 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40772 deadline: 1732486570422, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cdb3383a4019da5f5ef0568dfacd542e, server=bd6592c37bb2,40087,1732486423713 2024-11-24T22:15:10,425 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40087 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cdb3383a4019da5f5ef0568dfacd542e, server=bd6592c37bb2,40087,1732486423713 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-24T22:15:10,426 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732486488674.cdb3383a4019da5f5ef0568dfacd542e., hostname=bd6592c37bb2,40087,1732486423713, seqNum=2 , the old value is region=TestAcidGuarantees,,1732486488674.cdb3383a4019da5f5ef0568dfacd542e., hostname=bd6592c37bb2,40087,1732486423713, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cdb3383a4019da5f5ef0568dfacd542e, server=bd6592c37bb2,40087,1732486423713 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T22:15:10,426 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40087 {}] ipc.CallRunner(138): callId: 23 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40842 deadline: 1732486570422, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cdb3383a4019da5f5ef0568dfacd542e, server=bd6592c37bb2,40087,1732486423713 2024-11-24T22:15:10,426 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732486488674.cdb3383a4019da5f5ef0568dfacd542e., hostname=bd6592c37bb2,40087,1732486423713, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cdb3383a4019da5f5ef0568dfacd542e, server=bd6592c37bb2,40087,1732486423713 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T22:15:10,426 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732486488674.cdb3383a4019da5f5ef0568dfacd542e., hostname=bd6592c37bb2,40087,1732486423713, seqNum=2 because the exception is null or not the one we care about 2024-11-24T22:15:10,426 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40087 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cdb3383a4019da5f5ef0568dfacd542e, server=bd6592c37bb2,40087,1732486423713 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-24T22:15:10,426 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732486488674.cdb3383a4019da5f5ef0568dfacd542e., hostname=bd6592c37bb2,40087,1732486423713, seqNum=2 , the old value is region=TestAcidGuarantees,,1732486488674.cdb3383a4019da5f5ef0568dfacd542e., hostname=bd6592c37bb2,40087,1732486423713, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cdb3383a4019da5f5ef0568dfacd542e, server=bd6592c37bb2,40087,1732486423713 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T22:15:10,426 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40087 {}] ipc.CallRunner(138): callId: 27 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40806 deadline: 1732486570422, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cdb3383a4019da5f5ef0568dfacd542e, server=bd6592c37bb2,40087,1732486423713 2024-11-24T22:15:10,426 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732486488674.cdb3383a4019da5f5ef0568dfacd542e., hostname=bd6592c37bb2,40087,1732486423713, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cdb3383a4019da5f5ef0568dfacd542e, server=bd6592c37bb2,40087,1732486423713 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T22:15:10,426 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732486488674.cdb3383a4019da5f5ef0568dfacd542e., hostname=bd6592c37bb2,40087,1732486423713, seqNum=2 because the exception is null or not the one we care about 2024-11-24T22:15:10,426 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732486488674.cdb3383a4019da5f5ef0568dfacd542e., hostname=bd6592c37bb2,40087,1732486423713, seqNum=2 , the old value is region=TestAcidGuarantees,,1732486488674.cdb3383a4019da5f5ef0568dfacd542e., hostname=bd6592c37bb2,40087,1732486423713, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cdb3383a4019da5f5ef0568dfacd542e, server=bd6592c37bb2,40087,1732486423713 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T22:15:10,426 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732486488674.cdb3383a4019da5f5ef0568dfacd542e., hostname=bd6592c37bb2,40087,1732486423713, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cdb3383a4019da5f5ef0568dfacd542e, server=bd6592c37bb2,40087,1732486423713 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T22:15:10,426 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732486488674.cdb3383a4019da5f5ef0568dfacd542e., hostname=bd6592c37bb2,40087,1732486423713, seqNum=2 because the exception is null or not the one we care about 2024-11-24T22:15:10,443 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=40087 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=83 2024-11-24T22:15:10,443 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bd6592c37bb2:0-1 {event_type=RS_FLUSH_REGIONS, pid=83}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732486488674.cdb3383a4019da5f5ef0568dfacd542e. 2024-11-24T22:15:10,443 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bd6592c37bb2:0-1 {event_type=RS_FLUSH_REGIONS, pid=83}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732486488674.cdb3383a4019da5f5ef0568dfacd542e. as already flushing 2024-11-24T22:15:10,443 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bd6592c37bb2:0-1 {event_type=RS_FLUSH_REGIONS, pid=83}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732486488674.cdb3383a4019da5f5ef0568dfacd542e. 2024-11-24T22:15:10,443 ERROR [RS_FLUSH_OPERATIONS-regionserver/bd6592c37bb2:0-1 {event_type=RS_FLUSH_REGIONS, pid=83}] handler.RSProcedureHandler(58): pid=83 java.io.IOException: Unable to complete flush {ENCODED => cdb3383a4019da5f5ef0568dfacd542e, NAME => 'TestAcidGuarantees,,1732486488674.cdb3383a4019da5f5ef0568dfacd542e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T22:15:10,444 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bd6592c37bb2:0-1 {event_type=RS_FLUSH_REGIONS, pid=83}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=83 java.io.IOException: Unable to complete flush {ENCODED => cdb3383a4019da5f5ef0568dfacd542e, NAME => 'TestAcidGuarantees,,1732486488674.cdb3383a4019da5f5ef0568dfacd542e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T22:15:10,444 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37701 {}] master.HMaster(4177): Remote procedure failed, pid=83 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => cdb3383a4019da5f5ef0568dfacd542e, NAME => 'TestAcidGuarantees,,1732486488674.cdb3383a4019da5f5ef0568dfacd542e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => cdb3383a4019da5f5ef0568dfacd542e, NAME => 'TestAcidGuarantees,,1732486488674.cdb3383a4019da5f5ef0568dfacd542e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T22:15:10,524 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40087 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cdb3383a4019da5f5ef0568dfacd542e, server=bd6592c37bb2,40087,1732486423713 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-24T22:15:10,524 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40087 {}] ipc.CallRunner(138): callId: 401 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40834 deadline: 1732486570520, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cdb3383a4019da5f5ef0568dfacd542e, server=bd6592c37bb2,40087,1732486423713 2024-11-24T22:15:10,524 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732486488674.cdb3383a4019da5f5ef0568dfacd542e., hostname=bd6592c37bb2,40087,1732486423713, seqNum=2 , the old value is region=TestAcidGuarantees,,1732486488674.cdb3383a4019da5f5ef0568dfacd542e., hostname=bd6592c37bb2,40087,1732486423713, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cdb3383a4019da5f5ef0568dfacd542e, server=bd6592c37bb2,40087,1732486423713 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T22:15:10,524 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732486488674.cdb3383a4019da5f5ef0568dfacd542e., hostname=bd6592c37bb2,40087,1732486423713, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cdb3383a4019da5f5ef0568dfacd542e, server=bd6592c37bb2,40087,1732486423713 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T22:15:10,524 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732486488674.cdb3383a4019da5f5ef0568dfacd542e., hostname=bd6592c37bb2,40087,1732486423713, seqNum=2 because the exception is null or not the one we care about 2024-11-24T22:15:10,596 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=40087 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=83 2024-11-24T22:15:10,596 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bd6592c37bb2:0-2 {event_type=RS_FLUSH_REGIONS, pid=83}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732486488674.cdb3383a4019da5f5ef0568dfacd542e. 2024-11-24T22:15:10,596 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bd6592c37bb2:0-2 {event_type=RS_FLUSH_REGIONS, pid=83}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732486488674.cdb3383a4019da5f5ef0568dfacd542e. as already flushing 2024-11-24T22:15:10,596 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bd6592c37bb2:0-2 {event_type=RS_FLUSH_REGIONS, pid=83}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732486488674.cdb3383a4019da5f5ef0568dfacd542e. 2024-11-24T22:15:10,596 ERROR [RS_FLUSH_OPERATIONS-regionserver/bd6592c37bb2:0-2 {event_type=RS_FLUSH_REGIONS, pid=83}] handler.RSProcedureHandler(58): pid=83 java.io.IOException: Unable to complete flush {ENCODED => cdb3383a4019da5f5ef0568dfacd542e, NAME => 'TestAcidGuarantees,,1732486488674.cdb3383a4019da5f5ef0568dfacd542e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T22:15:10,597 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bd6592c37bb2:0-2 {event_type=RS_FLUSH_REGIONS, pid=83}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=83 java.io.IOException: Unable to complete flush {ENCODED => cdb3383a4019da5f5ef0568dfacd542e, NAME => 'TestAcidGuarantees,,1732486488674.cdb3383a4019da5f5ef0568dfacd542e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T22:15:10,597 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37701 {}] master.HMaster(4177): Remote procedure failed, pid=83 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => cdb3383a4019da5f5ef0568dfacd542e, NAME => 'TestAcidGuarantees,,1732486488674.cdb3383a4019da5f5ef0568dfacd542e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => cdb3383a4019da5f5ef0568dfacd542e, NAME => 'TestAcidGuarantees,,1732486488674.cdb3383a4019da5f5ef0568dfacd542e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T22:15:10,611 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37701 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=82 2024-11-24T22:15:10,663 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=465 (bloomFilter=true), to=hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/cdb3383a4019da5f5ef0568dfacd542e/.tmp/C/9dcfc731809446bdbe1c35518b60caee 2024-11-24T22:15:10,667 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/cdb3383a4019da5f5ef0568dfacd542e/.tmp/A/c60630f768c245218991d2174c071e39 as hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/cdb3383a4019da5f5ef0568dfacd542e/A/c60630f768c245218991d2174c071e39 2024-11-24T22:15:10,671 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/cdb3383a4019da5f5ef0568dfacd542e/A/c60630f768c245218991d2174c071e39, entries=200, sequenceid=465, filesize=14.4 K 2024-11-24T22:15:10,671 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/cdb3383a4019da5f5ef0568dfacd542e/.tmp/B/4e906e5b872246cba035629dd76e68f0 as hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/cdb3383a4019da5f5ef0568dfacd542e/B/4e906e5b872246cba035629dd76e68f0 2024-11-24T22:15:10,675 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/cdb3383a4019da5f5ef0568dfacd542e/B/4e906e5b872246cba035629dd76e68f0, entries=150, sequenceid=465, filesize=12.0 K 2024-11-24T22:15:10,676 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/cdb3383a4019da5f5ef0568dfacd542e/.tmp/C/9dcfc731809446bdbe1c35518b60caee as hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/cdb3383a4019da5f5ef0568dfacd542e/C/9dcfc731809446bdbe1c35518b60caee 2024-11-24T22:15:10,679 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/cdb3383a4019da5f5ef0568dfacd542e/C/9dcfc731809446bdbe1c35518b60caee, entries=150, sequenceid=465, filesize=12.0 K 2024-11-24T22:15:10,680 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~67.09 KB/68700, heapSize ~199.92 KB/204720, currentSize=107.34 KB/109920 for cdb3383a4019da5f5ef0568dfacd542e in 876ms, sequenceid=465, compaction requested=true 2024-11-24T22:15:10,680 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for cdb3383a4019da5f5ef0568dfacd542e: 2024-11-24T22:15:10,680 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store cdb3383a4019da5f5ef0568dfacd542e:A, priority=-2147483648, current under compaction store size is 1 2024-11-24T22:15:10,680 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-24T22:15:10,680 DEBUG [RS:0;bd6592c37bb2:40087-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-24T22:15:10,680 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store cdb3383a4019da5f5ef0568dfacd542e:B, priority=-2147483648, current under compaction store size is 2 2024-11-24T22:15:10,680 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-24T22:15:10,680 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store cdb3383a4019da5f5ef0568dfacd542e:C, priority=-2147483648, current under compaction store size is 3 2024-11-24T22:15:10,680 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-24T22:15:10,680 DEBUG [RS:0;bd6592c37bb2:40087-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-24T22:15:10,681 DEBUG [RS:0;bd6592c37bb2:40087-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 40331 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-24T22:15:10,681 DEBUG [RS:0;bd6592c37bb2:40087-shortCompactions-0 {}] regionserver.HStore(1541): cdb3383a4019da5f5ef0568dfacd542e/A is initiating minor compaction (all files) 2024-11-24T22:15:10,681 INFO [RS:0;bd6592c37bb2:40087-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of cdb3383a4019da5f5ef0568dfacd542e/A in TestAcidGuarantees,,1732486488674.cdb3383a4019da5f5ef0568dfacd542e. 2024-11-24T22:15:10,681 INFO [RS:0;bd6592c37bb2:40087-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/cdb3383a4019da5f5ef0568dfacd542e/A/7c55c2e8ae634c89bb30793907b35a80, hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/cdb3383a4019da5f5ef0568dfacd542e/A/930ac3ca836c49b0a3f4a7c6e9561a75, hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/cdb3383a4019da5f5ef0568dfacd542e/A/c60630f768c245218991d2174c071e39] into tmpdir=hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/cdb3383a4019da5f5ef0568dfacd542e/.tmp, totalSize=39.4 K 2024-11-24T22:15:10,681 DEBUG [RS:0;bd6592c37bb2:40087-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37891 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-24T22:15:10,682 DEBUG [RS:0;bd6592c37bb2:40087-longCompactions-0 {}] regionserver.HStore(1541): cdb3383a4019da5f5ef0568dfacd542e/B is initiating minor compaction (all files) 2024-11-24T22:15:10,682 INFO [RS:0;bd6592c37bb2:40087-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of cdb3383a4019da5f5ef0568dfacd542e/B in TestAcidGuarantees,,1732486488674.cdb3383a4019da5f5ef0568dfacd542e. 2024-11-24T22:15:10,682 INFO [RS:0;bd6592c37bb2:40087-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/cdb3383a4019da5f5ef0568dfacd542e/B/81e4f86e9c7d41eaa6187447800c6c2f, hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/cdb3383a4019da5f5ef0568dfacd542e/B/dc4df97aeba4401b8073d9a446f60dd5, hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/cdb3383a4019da5f5ef0568dfacd542e/B/4e906e5b872246cba035629dd76e68f0] into tmpdir=hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/cdb3383a4019da5f5ef0568dfacd542e/.tmp, totalSize=37.0 K 2024-11-24T22:15:10,682 DEBUG [RS:0;bd6592c37bb2:40087-shortCompactions-0 {}] compactions.Compactor(225): Compacting 7c55c2e8ae634c89bb30793907b35a80, keycount=150, bloomtype=ROW, size=13.0 K, encoding=NONE, compression=NONE, seqNum=429, earliestPutTs=1732486507206 2024-11-24T22:15:10,682 DEBUG [RS:0;bd6592c37bb2:40087-longCompactions-0 {}] compactions.Compactor(225): Compacting 81e4f86e9c7d41eaa6187447800c6c2f, keycount=150, bloomtype=ROW, size=13.0 K, encoding=NONE, compression=NONE, seqNum=429, earliestPutTs=1732486507206 2024-11-24T22:15:10,682 DEBUG [RS:0;bd6592c37bb2:40087-shortCompactions-0 {}] compactions.Compactor(225): Compacting 930ac3ca836c49b0a3f4a7c6e9561a75, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=449, earliestPutTs=1732486507604 2024-11-24T22:15:10,682 DEBUG [RS:0;bd6592c37bb2:40087-longCompactions-0 {}] compactions.Compactor(225): Compacting dc4df97aeba4401b8073d9a446f60dd5, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=449, earliestPutTs=1732486507604 2024-11-24T22:15:10,683 DEBUG [RS:0;bd6592c37bb2:40087-longCompactions-0 {}] compactions.Compactor(225): Compacting 4e906e5b872246cba035629dd76e68f0, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=465, earliestPutTs=1732486509265 2024-11-24T22:15:10,683 DEBUG [RS:0;bd6592c37bb2:40087-shortCompactions-0 {}] compactions.Compactor(225): Compacting c60630f768c245218991d2174c071e39, keycount=200, bloomtype=ROW, size=14.4 K, encoding=NONE, compression=NONE, seqNum=465, earliestPutTs=1732486509265 2024-11-24T22:15:10,695 INFO [RS:0;bd6592c37bb2:40087-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): cdb3383a4019da5f5ef0568dfacd542e#B#compaction#291 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-24T22:15:10,695 INFO [RS:0;bd6592c37bb2:40087-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): cdb3383a4019da5f5ef0568dfacd542e#A#compaction#292 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-24T22:15:10,695 DEBUG [RS:0;bd6592c37bb2:40087-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/cdb3383a4019da5f5ef0568dfacd542e/.tmp/B/4aeb173214884c898e278f7511d57ee5 is 50, key is test_row_0/B:col10/1732486509274/Put/seqid=0 2024-11-24T22:15:10,696 DEBUG [RS:0;bd6592c37bb2:40087-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/cdb3383a4019da5f5ef0568dfacd542e/.tmp/A/5dc71dd99901496ca0d627d3c126042a is 50, key is test_row_0/A:col10/1732486509274/Put/seqid=0 2024-11-24T22:15:10,736 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45929 is added to blk_1073742165_1341 (size=13391) 2024-11-24T22:15:10,744 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45929 is added to blk_1073742166_1342 (size=13391) 2024-11-24T22:15:10,749 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=40087 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=83 2024-11-24T22:15:10,749 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bd6592c37bb2:0-0 {event_type=RS_FLUSH_REGIONS, pid=83}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732486488674.cdb3383a4019da5f5ef0568dfacd542e. 2024-11-24T22:15:10,749 INFO [RS_FLUSH_OPERATIONS-regionserver/bd6592c37bb2:0-0 {event_type=RS_FLUSH_REGIONS, pid=83}] regionserver.HRegion(2902): Flushing cdb3383a4019da5f5ef0568dfacd542e 3/3 column families, dataSize=107.34 KB heapSize=319.50 KB 2024-11-24T22:15:10,750 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bd6592c37bb2:0-0 {event_type=RS_FLUSH_REGIONS, pid=83}] regionserver.CompactingMemStore(205): FLUSHING TO DISK cdb3383a4019da5f5ef0568dfacd542e, store=A 2024-11-24T22:15:10,750 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bd6592c37bb2:0-0 {event_type=RS_FLUSH_REGIONS, pid=83}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-24T22:15:10,750 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bd6592c37bb2:0-0 {event_type=RS_FLUSH_REGIONS, pid=83}] regionserver.CompactingMemStore(205): FLUSHING TO DISK cdb3383a4019da5f5ef0568dfacd542e, store=B 2024-11-24T22:15:10,750 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bd6592c37bb2:0-0 {event_type=RS_FLUSH_REGIONS, pid=83}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-24T22:15:10,750 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bd6592c37bb2:0-0 {event_type=RS_FLUSH_REGIONS, pid=83}] regionserver.CompactingMemStore(205): FLUSHING TO DISK cdb3383a4019da5f5ef0568dfacd542e, store=C 2024-11-24T22:15:10,750 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bd6592c37bb2:0-0 {event_type=RS_FLUSH_REGIONS, pid=83}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-24T22:15:10,754 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bd6592c37bb2:0-0 {event_type=RS_FLUSH_REGIONS, pid=83}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/cdb3383a4019da5f5ef0568dfacd542e/.tmp/A/8a68e80e8ed744a99ca3865b28ddfe00 is 50, key is test_row_0/A:col10/1732486509869/Put/seqid=0 2024-11-24T22:15:10,773 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45929 is added to blk_1073742167_1343 (size=12301) 2024-11-24T22:15:10,859 INFO [Thread-1152 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1152. 2024-11-24T22:15:10,859 DEBUG [Thread-1152 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicGetReader.workDone(AcidGuaranteesTestTool.java:234) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-24T22:15:10,859 DEBUG [Thread-1152 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-24T22:15:10,859 DEBUG [Thread-1152 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-24T22:15:10,860 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-24T22:15:10,860 INFO [Thread-1146 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1146. 2024-11-24T22:15:10,860 DEBUG [Thread-1146 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicGetReader.workDone(AcidGuaranteesTestTool.java:234) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-24T22:15:10,860 DEBUG [Thread-1146 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-24T22:15:10,861 DEBUG [Thread-1146 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-24T22:15:10,861 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-24T22:15:10,862 INFO [Thread-1148 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1148. 2024-11-24T22:15:10,862 DEBUG [Thread-1148 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicGetReader.workDone(AcidGuaranteesTestTool.java:234) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-24T22:15:10,862 DEBUG [Thread-1148 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-24T22:15:10,862 DEBUG [Thread-1148 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-24T22:15:10,862 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-24T22:15:10,863 INFO [Thread-1150 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1150. 2024-11-24T22:15:10,863 DEBUG [Thread-1150 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicGetReader.workDone(AcidGuaranteesTestTool.java:234) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-24T22:15:10,863 DEBUG [Thread-1150 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-24T22:15:10,863 DEBUG [Thread-1150 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-24T22:15:10,863 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-24T22:15:10,864 INFO [Thread-1154 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1154. 2024-11-24T22:15:10,864 DEBUG [Thread-1154 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicGetReader.workDone(AcidGuaranteesTestTool.java:234) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-24T22:15:10,864 DEBUG [Thread-1154 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-24T22:15:10,864 DEBUG [Thread-1154 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-24T22:15:10,864 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-24T22:15:11,032 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40087 {}] regionserver.HRegion(8855): Flush requested on cdb3383a4019da5f5ef0568dfacd542e 2024-11-24T22:15:11,032 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732486488674.cdb3383a4019da5f5ef0568dfacd542e. as already flushing 2024-11-24T22:15:11,032 INFO [Thread-1137 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1137. 2024-11-24T22:15:11,032 DEBUG [Thread-1137 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-24T22:15:11,032 DEBUG [Thread-1137 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-24T22:15:11,033 DEBUG [Thread-1137 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-24T22:15:11,033 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-24T22:15:11,120 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37701 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=82 2024-11-24T22:15:11,141 DEBUG [RS:0;bd6592c37bb2:40087-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/cdb3383a4019da5f5ef0568dfacd542e/.tmp/B/4aeb173214884c898e278f7511d57ee5 as hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/cdb3383a4019da5f5ef0568dfacd542e/B/4aeb173214884c898e278f7511d57ee5 2024-11-24T22:15:11,145 INFO [RS:0;bd6592c37bb2:40087-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in cdb3383a4019da5f5ef0568dfacd542e/B of cdb3383a4019da5f5ef0568dfacd542e into 4aeb173214884c898e278f7511d57ee5(size=13.1 K), total size for store is 13.1 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-24T22:15:11,145 DEBUG [RS:0;bd6592c37bb2:40087-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for cdb3383a4019da5f5ef0568dfacd542e: 2024-11-24T22:15:11,145 INFO [RS:0;bd6592c37bb2:40087-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732486488674.cdb3383a4019da5f5ef0568dfacd542e., storeName=cdb3383a4019da5f5ef0568dfacd542e/B, priority=13, startTime=1732486510680; duration=0sec 2024-11-24T22:15:11,145 DEBUG [RS:0;bd6592c37bb2:40087-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-24T22:15:11,145 DEBUG [RS:0;bd6592c37bb2:40087-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: cdb3383a4019da5f5ef0568dfacd542e:B 2024-11-24T22:15:11,146 DEBUG [RS:0;bd6592c37bb2:40087-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-24T22:15:11,146 DEBUG [RS:0;bd6592c37bb2:40087-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37891 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-24T22:15:11,146 DEBUG [RS:0;bd6592c37bb2:40087-longCompactions-0 {}] regionserver.HStore(1541): cdb3383a4019da5f5ef0568dfacd542e/C is initiating minor compaction (all files) 2024-11-24T22:15:11,146 INFO [RS:0;bd6592c37bb2:40087-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of cdb3383a4019da5f5ef0568dfacd542e/C in TestAcidGuarantees,,1732486488674.cdb3383a4019da5f5ef0568dfacd542e. 2024-11-24T22:15:11,147 INFO [RS:0;bd6592c37bb2:40087-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/cdb3383a4019da5f5ef0568dfacd542e/C/e522bb98c69848cf83b98d352440eb70, hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/cdb3383a4019da5f5ef0568dfacd542e/C/2bb428b726714735a7a3359a0464ba12, hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/cdb3383a4019da5f5ef0568dfacd542e/C/9dcfc731809446bdbe1c35518b60caee] into tmpdir=hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/cdb3383a4019da5f5ef0568dfacd542e/.tmp, totalSize=37.0 K 2024-11-24T22:15:11,147 DEBUG [RS:0;bd6592c37bb2:40087-longCompactions-0 {}] compactions.Compactor(225): Compacting e522bb98c69848cf83b98d352440eb70, keycount=150, bloomtype=ROW, size=13.0 K, encoding=NONE, compression=NONE, seqNum=429, earliestPutTs=1732486507206 2024-11-24T22:15:11,147 DEBUG [RS:0;bd6592c37bb2:40087-longCompactions-0 {}] compactions.Compactor(225): Compacting 2bb428b726714735a7a3359a0464ba12, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=449, earliestPutTs=1732486507604 2024-11-24T22:15:11,148 DEBUG [RS:0;bd6592c37bb2:40087-longCompactions-0 {}] compactions.Compactor(225): Compacting 9dcfc731809446bdbe1c35518b60caee, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=465, earliestPutTs=1732486509265 2024-11-24T22:15:11,148 DEBUG [RS:0;bd6592c37bb2:40087-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/cdb3383a4019da5f5ef0568dfacd542e/.tmp/A/5dc71dd99901496ca0d627d3c126042a as hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/cdb3383a4019da5f5ef0568dfacd542e/A/5dc71dd99901496ca0d627d3c126042a 2024-11-24T22:15:11,153 INFO [RS:0;bd6592c37bb2:40087-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in cdb3383a4019da5f5ef0568dfacd542e/A of cdb3383a4019da5f5ef0568dfacd542e into 5dc71dd99901496ca0d627d3c126042a(size=13.1 K), total size for store is 13.1 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-24T22:15:11,153 DEBUG [RS:0;bd6592c37bb2:40087-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for cdb3383a4019da5f5ef0568dfacd542e: 2024-11-24T22:15:11,153 INFO [RS:0;bd6592c37bb2:40087-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732486488674.cdb3383a4019da5f5ef0568dfacd542e., storeName=cdb3383a4019da5f5ef0568dfacd542e/A, priority=13, startTime=1732486510680; duration=0sec 2024-11-24T22:15:11,153 DEBUG [RS:0;bd6592c37bb2:40087-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-24T22:15:11,153 DEBUG [RS:0;bd6592c37bb2:40087-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: cdb3383a4019da5f5ef0568dfacd542e:A 2024-11-24T22:15:11,153 INFO [RS:0;bd6592c37bb2:40087-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): cdb3383a4019da5f5ef0568dfacd542e#C#compaction#294 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-24T22:15:11,154 DEBUG [RS:0;bd6592c37bb2:40087-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/cdb3383a4019da5f5ef0568dfacd542e/.tmp/C/74528b20d68f43b897f2d61fe8f9f500 is 50, key is test_row_0/C:col10/1732486509274/Put/seqid=0 2024-11-24T22:15:11,157 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45929 is added to blk_1073742168_1344 (size=13391) 2024-11-24T22:15:11,174 INFO [RS_FLUSH_OPERATIONS-regionserver/bd6592c37bb2:0-0 {event_type=RS_FLUSH_REGIONS, pid=83}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=35.78 KB at sequenceid=484 (bloomFilter=true), to=hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/cdb3383a4019da5f5ef0568dfacd542e/.tmp/A/8a68e80e8ed744a99ca3865b28ddfe00 2024-11-24T22:15:11,180 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bd6592c37bb2:0-0 {event_type=RS_FLUSH_REGIONS, pid=83}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/cdb3383a4019da5f5ef0568dfacd542e/.tmp/B/a2bb898be65d4dd3b1b02687a0ba9245 is 50, key is test_row_0/B:col10/1732486509869/Put/seqid=0 2024-11-24T22:15:11,183 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45929 is added to blk_1073742169_1345 (size=12301) 2024-11-24T22:15:11,432 INFO [Thread-1135 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1135. 2024-11-24T22:15:11,432 DEBUG [Thread-1135 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-24T22:15:11,432 DEBUG [Thread-1135 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-24T22:15:11,432 DEBUG [Thread-1135 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-24T22:15:11,432 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-24T22:15:11,443 INFO [Thread-1139 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1139. 2024-11-24T22:15:11,443 INFO [Thread-1143 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1143. 2024-11-24T22:15:11,443 DEBUG [Thread-1139 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-24T22:15:11,443 DEBUG [Thread-1139 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-24T22:15:11,443 DEBUG [Thread-1143 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-24T22:15:11,443 DEBUG [Thread-1143 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-24T22:15:11,443 DEBUG [Thread-1139 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-24T22:15:11,443 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-24T22:15:11,443 DEBUG [Thread-1143 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-24T22:15:11,443 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-24T22:15:11,561 DEBUG [RS:0;bd6592c37bb2:40087-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/cdb3383a4019da5f5ef0568dfacd542e/.tmp/C/74528b20d68f43b897f2d61fe8f9f500 as hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/cdb3383a4019da5f5ef0568dfacd542e/C/74528b20d68f43b897f2d61fe8f9f500 2024-11-24T22:15:11,565 INFO [RS:0;bd6592c37bb2:40087-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in cdb3383a4019da5f5ef0568dfacd542e/C of cdb3383a4019da5f5ef0568dfacd542e into 74528b20d68f43b897f2d61fe8f9f500(size=13.1 K), total size for store is 13.1 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-24T22:15:11,565 DEBUG [RS:0;bd6592c37bb2:40087-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for cdb3383a4019da5f5ef0568dfacd542e: 2024-11-24T22:15:11,565 INFO [RS:0;bd6592c37bb2:40087-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732486488674.cdb3383a4019da5f5ef0568dfacd542e., storeName=cdb3383a4019da5f5ef0568dfacd542e/C, priority=13, startTime=1732486510680; duration=0sec 2024-11-24T22:15:11,565 DEBUG [RS:0;bd6592c37bb2:40087-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-24T22:15:11,565 DEBUG [RS:0;bd6592c37bb2:40087-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: cdb3383a4019da5f5ef0568dfacd542e:C 2024-11-24T22:15:11,584 INFO [RS_FLUSH_OPERATIONS-regionserver/bd6592c37bb2:0-0 {event_type=RS_FLUSH_REGIONS, pid=83}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=35.78 KB at sequenceid=484 (bloomFilter=true), to=hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/cdb3383a4019da5f5ef0568dfacd542e/.tmp/B/a2bb898be65d4dd3b1b02687a0ba9245 2024-11-24T22:15:11,590 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bd6592c37bb2:0-0 {event_type=RS_FLUSH_REGIONS, pid=83}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/cdb3383a4019da5f5ef0568dfacd542e/.tmp/C/3fa8093123c44267bf96cab6f4dbd200 is 50, key is test_row_0/C:col10/1732486509869/Put/seqid=0 2024-11-24T22:15:11,593 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45929 is added to blk_1073742170_1346 (size=12301) 2024-11-24T22:15:11,993 INFO [RS_FLUSH_OPERATIONS-regionserver/bd6592c37bb2:0-0 {event_type=RS_FLUSH_REGIONS, pid=83}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=35.78 KB at sequenceid=484 (bloomFilter=true), to=hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/cdb3383a4019da5f5ef0568dfacd542e/.tmp/C/3fa8093123c44267bf96cab6f4dbd200 2024-11-24T22:15:11,997 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bd6592c37bb2:0-0 {event_type=RS_FLUSH_REGIONS, pid=83}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/cdb3383a4019da5f5ef0568dfacd542e/.tmp/A/8a68e80e8ed744a99ca3865b28ddfe00 as hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/cdb3383a4019da5f5ef0568dfacd542e/A/8a68e80e8ed744a99ca3865b28ddfe00 2024-11-24T22:15:12,000 INFO [RS_FLUSH_OPERATIONS-regionserver/bd6592c37bb2:0-0 {event_type=RS_FLUSH_REGIONS, pid=83}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/cdb3383a4019da5f5ef0568dfacd542e/A/8a68e80e8ed744a99ca3865b28ddfe00, entries=150, sequenceid=484, filesize=12.0 K 2024-11-24T22:15:12,001 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bd6592c37bb2:0-0 {event_type=RS_FLUSH_REGIONS, pid=83}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/cdb3383a4019da5f5ef0568dfacd542e/.tmp/B/a2bb898be65d4dd3b1b02687a0ba9245 as hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/cdb3383a4019da5f5ef0568dfacd542e/B/a2bb898be65d4dd3b1b02687a0ba9245 2024-11-24T22:15:12,004 INFO [RS_FLUSH_OPERATIONS-regionserver/bd6592c37bb2:0-0 {event_type=RS_FLUSH_REGIONS, pid=83}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/cdb3383a4019da5f5ef0568dfacd542e/B/a2bb898be65d4dd3b1b02687a0ba9245, entries=150, sequenceid=484, filesize=12.0 K 2024-11-24T22:15:12,004 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bd6592c37bb2:0-0 {event_type=RS_FLUSH_REGIONS, pid=83}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/cdb3383a4019da5f5ef0568dfacd542e/.tmp/C/3fa8093123c44267bf96cab6f4dbd200 as hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/cdb3383a4019da5f5ef0568dfacd542e/C/3fa8093123c44267bf96cab6f4dbd200 2024-11-24T22:15:12,007 INFO [RS_FLUSH_OPERATIONS-regionserver/bd6592c37bb2:0-0 {event_type=RS_FLUSH_REGIONS, pid=83}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/cdb3383a4019da5f5ef0568dfacd542e/C/3fa8093123c44267bf96cab6f4dbd200, entries=150, sequenceid=484, filesize=12.0 K 2024-11-24T22:15:12,008 INFO [RS_FLUSH_OPERATIONS-regionserver/bd6592c37bb2:0-0 {event_type=RS_FLUSH_REGIONS, pid=83}] regionserver.HRegion(3140): Finished flush of dataSize ~107.34 KB/109920, heapSize ~319.45 KB/327120, currentSize=26.84 KB/27480 for cdb3383a4019da5f5ef0568dfacd542e in 1259ms, sequenceid=484, compaction requested=false 2024-11-24T22:15:12,008 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bd6592c37bb2:0-0 {event_type=RS_FLUSH_REGIONS, pid=83}] regionserver.HRegion(2603): Flush status journal for cdb3383a4019da5f5ef0568dfacd542e: 2024-11-24T22:15:12,008 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bd6592c37bb2:0-0 {event_type=RS_FLUSH_REGIONS, pid=83}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732486488674.cdb3383a4019da5f5ef0568dfacd542e. 2024-11-24T22:15:12,008 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bd6592c37bb2:0-0 {event_type=RS_FLUSH_REGIONS, pid=83}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=83 2024-11-24T22:15:12,008 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37701 {}] master.HMaster(4169): Remote procedure done, pid=83 2024-11-24T22:15:12,010 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=83, resume processing ppid=82 2024-11-24T22:15:12,010 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=83, ppid=82, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 2.0250 sec 2024-11-24T22:15:12,012 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=82, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=82, table=TestAcidGuarantees in 2.0290 sec 2024-11-24T22:15:12,015 DEBUG [FsDatasetAsyncDiskServiceFixer {}] hbase.HBaseTestingUtil$FsDatasetAsyncDiskServiceFixer(597): NoSuchFieldException: threadGroup; It might because your Hadoop version > 3.2.3 or 3.3.4, See HBASE-27595 for details. 2024-11-24T22:15:12,131 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37701 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=82 2024-11-24T22:15:12,131 INFO [RPCClient-NioEventLoopGroup-4-6 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-24T22:15:19,302 INFO [Thread-1141 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1141. 2024-11-24T22:15:19,302 DEBUG [Thread-1141 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-24T22:15:19,302 DEBUG [Thread-1141 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-24T22:15:19,302 DEBUG [Thread-1141 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-24T22:15:19,302 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-24T22:15:19,303 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(392): Finished test. Writers: 2024-11-24T22:15:19,303 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 15 2024-11-24T22:15:19,303 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 339 2024-11-24T22:15:19,303 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 11 2024-11-24T22:15:19,303 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 10 2024-11-24T22:15:19,303 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 14 2024-11-24T22:15:19,303 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(396): Readers: 2024-11-24T22:15:19,303 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(398): read 3082 2024-11-24T22:15:19,303 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(398): read 3074 2024-11-24T22:15:19,303 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(398): read 3160 2024-11-24T22:15:19,303 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(398): read 3086 2024-11-24T22:15:19,303 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(398): read 3081 2024-11-24T22:15:19,303 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(400): Scanners: 2024-11-24T22:15:19,303 INFO [Time-limited test {}] client.AsyncConnectionImpl(233): Connection has been closed by Time-limited test. 2024-11-24T22:15:19,303 DEBUG [Time-limited test {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool.doWork(AcidGuaranteesTestTool.java:131) at org.apache.hadoop.hbase.util.AbstractHBaseTool.run(AbstractHBaseTool.java:150) at org.apache.hadoop.hbase.AcidGuaranteesTestBase.runTestAtomicity(AcidGuaranteesTestBase.java:102) at org.apache.hadoop.hbase.AcidGuaranteesTestBase.runTestAtomicity(AcidGuaranteesTestBase.java:91) at org.apache.hadoop.hbase.AcidGuaranteesTestBase.testGetAtomicity(AcidGuaranteesTestBase.java:107) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) at java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) at java.base/java.lang.reflect.Method.invoke(Method.java:568) at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) at java.base/java.util.concurrent.FutureTask.run(FutureTask.java:264) at java.base/java.lang.Thread.run(Thread.java:840) 2024-11-24T22:15:19,303 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-24T22:15:19,303 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-24T22:15:19,303 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-24T22:15:19,304 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37701 {}] master.HMaster$13(2820): Client=jenkins//172.17.0.2 disable TestAcidGuarantees 2024-11-24T22:15:19,305 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37701 {}] procedure2.ProcedureExecutor(1139): Stored pid=84, state=RUNNABLE:DISABLE_TABLE_PREPARE, hasLock=false; DisableTableProcedure table=TestAcidGuarantees 2024-11-24T22:15:19,306 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37701 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=84 2024-11-24T22:15:19,306 DEBUG [PEWorker-3 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1732486519306"}]},"ts":"1732486519306"} 2024-11-24T22:15:19,308 INFO [PEWorker-3 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=DISABLING in hbase:meta 2024-11-24T22:15:19,308 INFO [PEWorker-3 {}] procedure.DisableTableProcedure(284): Set TestAcidGuarantees to state=DISABLING 2024-11-24T22:15:19,308 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=85, ppid=84, state=RUNNABLE:CLOSE_TABLE_REGIONS_SCHEDULE, hasLock=false; CloseTableRegionsProcedure table=TestAcidGuarantees}] 2024-11-24T22:15:19,309 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=86, ppid=85, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=cdb3383a4019da5f5ef0568dfacd542e, UNASSIGN}] 2024-11-24T22:15:19,310 INFO [PEWorker-2 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=86, ppid=85, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=cdb3383a4019da5f5ef0568dfacd542e, UNASSIGN 2024-11-24T22:15:19,311 INFO [PEWorker-2 {}] assignment.RegionStateStore(223): pid=86 updating hbase:meta row=cdb3383a4019da5f5ef0568dfacd542e, regionState=CLOSING, regionLocation=bd6592c37bb2,40087,1732486423713 2024-11-24T22:15:19,312 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=86, ppid=85, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=cdb3383a4019da5f5ef0568dfacd542e, UNASSIGN because future has completed 2024-11-24T22:15:19,312 DEBUG [PEWorker-4 {}] assignment.TransitRegionStateProcedure(375): Close region: isSplit: false: evictOnSplit: true: evictOnClose: false 2024-11-24T22:15:19,312 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=87, ppid=86, state=RUNNABLE, hasLock=false; CloseRegionProcedure cdb3383a4019da5f5ef0568dfacd542e, server=bd6592c37bb2,40087,1732486423713}] 2024-11-24T22:15:19,410 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37701 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=84 2024-11-24T22:15:19,464 INFO [RS_CLOSE_REGION-regionserver/bd6592c37bb2:0-0 {event_type=M_RS_CLOSE_REGION, pid=87}] handler.UnassignRegionHandler(122): Close cdb3383a4019da5f5ef0568dfacd542e 2024-11-24T22:15:19,464 DEBUG [RS_CLOSE_REGION-regionserver/bd6592c37bb2:0-0 {event_type=M_RS_CLOSE_REGION, pid=87}] handler.UnassignRegionHandler(136): Unassign region: split region: false: evictCache: false 2024-11-24T22:15:19,465 DEBUG [RS_CLOSE_REGION-regionserver/bd6592c37bb2:0-0 {event_type=M_RS_CLOSE_REGION, pid=87}] regionserver.HRegion(1722): Closing cdb3383a4019da5f5ef0568dfacd542e, disabling compactions & flushes 2024-11-24T22:15:19,465 INFO [RS_CLOSE_REGION-regionserver/bd6592c37bb2:0-0 {event_type=M_RS_CLOSE_REGION, pid=87}] regionserver.HRegion(1755): Closing region TestAcidGuarantees,,1732486488674.cdb3383a4019da5f5ef0568dfacd542e. 2024-11-24T22:15:19,465 DEBUG [RS_CLOSE_REGION-regionserver/bd6592c37bb2:0-0 {event_type=M_RS_CLOSE_REGION, pid=87}] regionserver.HRegion(1776): Time limited wait for close lock on TestAcidGuarantees,,1732486488674.cdb3383a4019da5f5ef0568dfacd542e. 2024-11-24T22:15:19,465 DEBUG [RS_CLOSE_REGION-regionserver/bd6592c37bb2:0-0 {event_type=M_RS_CLOSE_REGION, pid=87}] regionserver.HRegion(1843): Acquired close lock on TestAcidGuarantees,,1732486488674.cdb3383a4019da5f5ef0568dfacd542e. after waiting 0 ms 2024-11-24T22:15:19,465 DEBUG [RS_CLOSE_REGION-regionserver/bd6592c37bb2:0-0 {event_type=M_RS_CLOSE_REGION, pid=87}] regionserver.HRegion(1853): Updates disabled for region TestAcidGuarantees,,1732486488674.cdb3383a4019da5f5ef0568dfacd542e. 2024-11-24T22:15:19,465 INFO [RS_CLOSE_REGION-regionserver/bd6592c37bb2:0-0 {event_type=M_RS_CLOSE_REGION, pid=87}] regionserver.HRegion(2902): Flushing cdb3383a4019da5f5ef0568dfacd542e 3/3 column families, dataSize=33.54 KB heapSize=100.36 KB 2024-11-24T22:15:19,465 DEBUG [RS_CLOSE_REGION-regionserver/bd6592c37bb2:0-0 {event_type=M_RS_CLOSE_REGION, pid=87}] regionserver.CompactingMemStore(205): FLUSHING TO DISK cdb3383a4019da5f5ef0568dfacd542e, store=A 2024-11-24T22:15:19,465 DEBUG [RS_CLOSE_REGION-regionserver/bd6592c37bb2:0-0 {event_type=M_RS_CLOSE_REGION, pid=87}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-24T22:15:19,465 DEBUG [RS_CLOSE_REGION-regionserver/bd6592c37bb2:0-0 {event_type=M_RS_CLOSE_REGION, pid=87}] regionserver.CompactingMemStore(205): FLUSHING TO DISK cdb3383a4019da5f5ef0568dfacd542e, store=B 2024-11-24T22:15:19,465 DEBUG [RS_CLOSE_REGION-regionserver/bd6592c37bb2:0-0 {event_type=M_RS_CLOSE_REGION, pid=87}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-24T22:15:19,465 DEBUG [RS_CLOSE_REGION-regionserver/bd6592c37bb2:0-0 {event_type=M_RS_CLOSE_REGION, pid=87}] regionserver.CompactingMemStore(205): FLUSHING TO DISK cdb3383a4019da5f5ef0568dfacd542e, store=C 2024-11-24T22:15:19,465 DEBUG [RS_CLOSE_REGION-regionserver/bd6592c37bb2:0-0 {event_type=M_RS_CLOSE_REGION, pid=87}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-24T22:15:19,469 DEBUG [RS_CLOSE_REGION-regionserver/bd6592c37bb2:0-0 {event_type=M_RS_CLOSE_REGION, pid=87}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/cdb3383a4019da5f5ef0568dfacd542e/.tmp/A/cb02947752e140208717d48bcffc96e9 is 50, key is test_row_0/A:col10/1732486511441/Put/seqid=0 2024-11-24T22:15:19,472 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45929 is added to blk_1073742171_1347 (size=12301) 2024-11-24T22:15:19,621 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37701 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=84 2024-11-24T22:15:19,873 INFO [RS_CLOSE_REGION-regionserver/bd6592c37bb2:0-0 {event_type=M_RS_CLOSE_REGION, pid=87}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=11.18 KB at sequenceid=495 (bloomFilter=true), to=hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/cdb3383a4019da5f5ef0568dfacd542e/.tmp/A/cb02947752e140208717d48bcffc96e9 2024-11-24T22:15:19,883 DEBUG [RS_CLOSE_REGION-regionserver/bd6592c37bb2:0-0 {event_type=M_RS_CLOSE_REGION, pid=87}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/cdb3383a4019da5f5ef0568dfacd542e/.tmp/B/4dda70a002ac4948bc84c4e9e55279ee is 50, key is test_row_0/B:col10/1732486511441/Put/seqid=0 2024-11-24T22:15:19,902 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45929 is added to blk_1073742172_1348 (size=12301) 2024-11-24T22:15:19,903 INFO [RS_CLOSE_REGION-regionserver/bd6592c37bb2:0-0 {event_type=M_RS_CLOSE_REGION, pid=87}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=11.18 KB at sequenceid=495 (bloomFilter=true), to=hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/cdb3383a4019da5f5ef0568dfacd542e/.tmp/B/4dda70a002ac4948bc84c4e9e55279ee 2024-11-24T22:15:19,912 DEBUG [RS_CLOSE_REGION-regionserver/bd6592c37bb2:0-0 {event_type=M_RS_CLOSE_REGION, pid=87}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/cdb3383a4019da5f5ef0568dfacd542e/.tmp/C/9f1fc6c1560e45578c0ce54cee5c4bca is 50, key is test_row_0/C:col10/1732486511441/Put/seqid=0 2024-11-24T22:15:19,918 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45929 is added to blk_1073742173_1349 (size=12301) 2024-11-24T22:15:19,931 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37701 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=84 2024-11-24T22:15:20,319 INFO [RS_CLOSE_REGION-regionserver/bd6592c37bb2:0-0 {event_type=M_RS_CLOSE_REGION, pid=87}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=11.18 KB at sequenceid=495 (bloomFilter=true), to=hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/cdb3383a4019da5f5ef0568dfacd542e/.tmp/C/9f1fc6c1560e45578c0ce54cee5c4bca 2024-11-24T22:15:20,325 DEBUG [RS_CLOSE_REGION-regionserver/bd6592c37bb2:0-0 {event_type=M_RS_CLOSE_REGION, pid=87}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/cdb3383a4019da5f5ef0568dfacd542e/.tmp/A/cb02947752e140208717d48bcffc96e9 as hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/cdb3383a4019da5f5ef0568dfacd542e/A/cb02947752e140208717d48bcffc96e9 2024-11-24T22:15:20,329 INFO [RS_CLOSE_REGION-regionserver/bd6592c37bb2:0-0 {event_type=M_RS_CLOSE_REGION, pid=87}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/cdb3383a4019da5f5ef0568dfacd542e/A/cb02947752e140208717d48bcffc96e9, entries=150, sequenceid=495, filesize=12.0 K 2024-11-24T22:15:20,329 DEBUG [RS_CLOSE_REGION-regionserver/bd6592c37bb2:0-0 {event_type=M_RS_CLOSE_REGION, pid=87}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/cdb3383a4019da5f5ef0568dfacd542e/.tmp/B/4dda70a002ac4948bc84c4e9e55279ee as hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/cdb3383a4019da5f5ef0568dfacd542e/B/4dda70a002ac4948bc84c4e9e55279ee 2024-11-24T22:15:20,333 INFO [RS_CLOSE_REGION-regionserver/bd6592c37bb2:0-0 {event_type=M_RS_CLOSE_REGION, pid=87}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/cdb3383a4019da5f5ef0568dfacd542e/B/4dda70a002ac4948bc84c4e9e55279ee, entries=150, sequenceid=495, filesize=12.0 K 2024-11-24T22:15:20,334 DEBUG [RS_CLOSE_REGION-regionserver/bd6592c37bb2:0-0 {event_type=M_RS_CLOSE_REGION, pid=87}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/cdb3383a4019da5f5ef0568dfacd542e/.tmp/C/9f1fc6c1560e45578c0ce54cee5c4bca as hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/cdb3383a4019da5f5ef0568dfacd542e/C/9f1fc6c1560e45578c0ce54cee5c4bca 2024-11-24T22:15:20,337 INFO [RS_CLOSE_REGION-regionserver/bd6592c37bb2:0-0 {event_type=M_RS_CLOSE_REGION, pid=87}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/cdb3383a4019da5f5ef0568dfacd542e/C/9f1fc6c1560e45578c0ce54cee5c4bca, entries=150, sequenceid=495, filesize=12.0 K 2024-11-24T22:15:20,338 INFO [RS_CLOSE_REGION-regionserver/bd6592c37bb2:0-0 {event_type=M_RS_CLOSE_REGION, pid=87}] regionserver.HRegion(3140): Finished flush of dataSize ~33.54 KB/34350, heapSize ~100.31 KB/102720, currentSize=0 B/0 for cdb3383a4019da5f5ef0568dfacd542e in 873ms, sequenceid=495, compaction requested=true 2024-11-24T22:15:20,338 DEBUG [StoreCloser-TestAcidGuarantees,,1732486488674.cdb3383a4019da5f5ef0568dfacd542e.-1 {}] regionserver.HStore(2317): Moving the files [hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/cdb3383a4019da5f5ef0568dfacd542e/A/4eb6e5cbfdd146468d5f9e4bee6f4417, hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/cdb3383a4019da5f5ef0568dfacd542e/A/afbf7431336d498ab5f0bde061ac8ef3, hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/cdb3383a4019da5f5ef0568dfacd542e/A/278445a736f74720b5568544cc19082a, hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/cdb3383a4019da5f5ef0568dfacd542e/A/7f34a3a979ba46fb954a392c6945ce2f, hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/cdb3383a4019da5f5ef0568dfacd542e/A/aa7e9dc47e4f4e2ebe297d6e390500bd, hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/cdb3383a4019da5f5ef0568dfacd542e/A/516acb04f2084f5ab2fb65855735bfac, hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/cdb3383a4019da5f5ef0568dfacd542e/A/97b091d10a0d4c3e9b134e1fc633706f, hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/cdb3383a4019da5f5ef0568dfacd542e/A/20eee9fcfcec4fceab01ad537875e97e, hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/cdb3383a4019da5f5ef0568dfacd542e/A/fa66f306f0f04c7d8adb9bd8978e52a5, hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/cdb3383a4019da5f5ef0568dfacd542e/A/a05175f60c2046c8a2f1285b11374431, hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/cdb3383a4019da5f5ef0568dfacd542e/A/7ad153faedb44da490fdd7d12b6ba07b, hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/cdb3383a4019da5f5ef0568dfacd542e/A/34f1f60d5fbd41e9a65ea799007f6da7, hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/cdb3383a4019da5f5ef0568dfacd542e/A/ca4d693f159f40c89368d00de5a37289, hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/cdb3383a4019da5f5ef0568dfacd542e/A/acf5a3967a5f4ec6b719f46f93df16f7, hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/cdb3383a4019da5f5ef0568dfacd542e/A/6a5083c0ee774458bf24c31fed15ca5c, hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/cdb3383a4019da5f5ef0568dfacd542e/A/79b97ac10a544c658a9ec43f5d94fe7f, hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/cdb3383a4019da5f5ef0568dfacd542e/A/fd12ade02bb949afabf88523cee4fd62, hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/cdb3383a4019da5f5ef0568dfacd542e/A/e186d6bee8e04f47944534a07e432d52, hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/cdb3383a4019da5f5ef0568dfacd542e/A/22d2943f9286475fbf20ee37ad382e10, hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/cdb3383a4019da5f5ef0568dfacd542e/A/432138b440f142868293ae834c1ed131, hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/cdb3383a4019da5f5ef0568dfacd542e/A/ff0660ba25d7415a932d59d3791e2c25, hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/cdb3383a4019da5f5ef0568dfacd542e/A/9db012e14a6642439d367bf3ad8fe408, hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/cdb3383a4019da5f5ef0568dfacd542e/A/d5f5c9e33fef41fc8ef8df16567a8fe8, hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/cdb3383a4019da5f5ef0568dfacd542e/A/92e9d617735f4590b5aa7359e2f87d5e, hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/cdb3383a4019da5f5ef0568dfacd542e/A/9134c9ad987e430695cb0d7651a28586, hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/cdb3383a4019da5f5ef0568dfacd542e/A/738c0933918f47e68f1f385ea5ede3d9, hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/cdb3383a4019da5f5ef0568dfacd542e/A/110c91754e0a45988a2018e87fa8d232, hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/cdb3383a4019da5f5ef0568dfacd542e/A/ea40fb6886784ae982e1a5b7960a5111, hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/cdb3383a4019da5f5ef0568dfacd542e/A/7c55c2e8ae634c89bb30793907b35a80, hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/cdb3383a4019da5f5ef0568dfacd542e/A/0518f72a50f24932b904aa7048e69b43, hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/cdb3383a4019da5f5ef0568dfacd542e/A/930ac3ca836c49b0a3f4a7c6e9561a75, hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/cdb3383a4019da5f5ef0568dfacd542e/A/c60630f768c245218991d2174c071e39] to archive 2024-11-24T22:15:20,339 DEBUG [StoreCloser-TestAcidGuarantees,,1732486488674.cdb3383a4019da5f5ef0568dfacd542e.-1 {}] backup.HFileArchiver(360): Archiving compacted files. 2024-11-24T22:15:20,341 DEBUG [StoreCloser-TestAcidGuarantees,,1732486488674.cdb3383a4019da5f5ef0568dfacd542e.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/cdb3383a4019da5f5ef0568dfacd542e/A/4eb6e5cbfdd146468d5f9e4bee6f4417 to hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/archive/data/default/TestAcidGuarantees/cdb3383a4019da5f5ef0568dfacd542e/A/4eb6e5cbfdd146468d5f9e4bee6f4417 2024-11-24T22:15:20,342 DEBUG [StoreCloser-TestAcidGuarantees,,1732486488674.cdb3383a4019da5f5ef0568dfacd542e.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/cdb3383a4019da5f5ef0568dfacd542e/A/afbf7431336d498ab5f0bde061ac8ef3 to hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/archive/data/default/TestAcidGuarantees/cdb3383a4019da5f5ef0568dfacd542e/A/afbf7431336d498ab5f0bde061ac8ef3 2024-11-24T22:15:20,343 DEBUG [StoreCloser-TestAcidGuarantees,,1732486488674.cdb3383a4019da5f5ef0568dfacd542e.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/cdb3383a4019da5f5ef0568dfacd542e/A/278445a736f74720b5568544cc19082a to hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/archive/data/default/TestAcidGuarantees/cdb3383a4019da5f5ef0568dfacd542e/A/278445a736f74720b5568544cc19082a 2024-11-24T22:15:20,344 DEBUG [StoreCloser-TestAcidGuarantees,,1732486488674.cdb3383a4019da5f5ef0568dfacd542e.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/cdb3383a4019da5f5ef0568dfacd542e/A/7f34a3a979ba46fb954a392c6945ce2f to hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/archive/data/default/TestAcidGuarantees/cdb3383a4019da5f5ef0568dfacd542e/A/7f34a3a979ba46fb954a392c6945ce2f 2024-11-24T22:15:20,345 DEBUG [StoreCloser-TestAcidGuarantees,,1732486488674.cdb3383a4019da5f5ef0568dfacd542e.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/cdb3383a4019da5f5ef0568dfacd542e/A/aa7e9dc47e4f4e2ebe297d6e390500bd to hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/archive/data/default/TestAcidGuarantees/cdb3383a4019da5f5ef0568dfacd542e/A/aa7e9dc47e4f4e2ebe297d6e390500bd 2024-11-24T22:15:20,345 DEBUG [StoreCloser-TestAcidGuarantees,,1732486488674.cdb3383a4019da5f5ef0568dfacd542e.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/cdb3383a4019da5f5ef0568dfacd542e/A/516acb04f2084f5ab2fb65855735bfac to hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/archive/data/default/TestAcidGuarantees/cdb3383a4019da5f5ef0568dfacd542e/A/516acb04f2084f5ab2fb65855735bfac 2024-11-24T22:15:20,346 DEBUG [StoreCloser-TestAcidGuarantees,,1732486488674.cdb3383a4019da5f5ef0568dfacd542e.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/cdb3383a4019da5f5ef0568dfacd542e/A/97b091d10a0d4c3e9b134e1fc633706f to hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/archive/data/default/TestAcidGuarantees/cdb3383a4019da5f5ef0568dfacd542e/A/97b091d10a0d4c3e9b134e1fc633706f 2024-11-24T22:15:20,347 DEBUG [StoreCloser-TestAcidGuarantees,,1732486488674.cdb3383a4019da5f5ef0568dfacd542e.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/cdb3383a4019da5f5ef0568dfacd542e/A/20eee9fcfcec4fceab01ad537875e97e to hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/archive/data/default/TestAcidGuarantees/cdb3383a4019da5f5ef0568dfacd542e/A/20eee9fcfcec4fceab01ad537875e97e 2024-11-24T22:15:20,348 DEBUG [StoreCloser-TestAcidGuarantees,,1732486488674.cdb3383a4019da5f5ef0568dfacd542e.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/cdb3383a4019da5f5ef0568dfacd542e/A/fa66f306f0f04c7d8adb9bd8978e52a5 to hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/archive/data/default/TestAcidGuarantees/cdb3383a4019da5f5ef0568dfacd542e/A/fa66f306f0f04c7d8adb9bd8978e52a5 2024-11-24T22:15:20,349 DEBUG [StoreCloser-TestAcidGuarantees,,1732486488674.cdb3383a4019da5f5ef0568dfacd542e.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/cdb3383a4019da5f5ef0568dfacd542e/A/a05175f60c2046c8a2f1285b11374431 to hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/archive/data/default/TestAcidGuarantees/cdb3383a4019da5f5ef0568dfacd542e/A/a05175f60c2046c8a2f1285b11374431 2024-11-24T22:15:20,350 DEBUG [StoreCloser-TestAcidGuarantees,,1732486488674.cdb3383a4019da5f5ef0568dfacd542e.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/cdb3383a4019da5f5ef0568dfacd542e/A/7ad153faedb44da490fdd7d12b6ba07b to hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/archive/data/default/TestAcidGuarantees/cdb3383a4019da5f5ef0568dfacd542e/A/7ad153faedb44da490fdd7d12b6ba07b 2024-11-24T22:15:20,350 DEBUG [StoreCloser-TestAcidGuarantees,,1732486488674.cdb3383a4019da5f5ef0568dfacd542e.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/cdb3383a4019da5f5ef0568dfacd542e/A/34f1f60d5fbd41e9a65ea799007f6da7 to hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/archive/data/default/TestAcidGuarantees/cdb3383a4019da5f5ef0568dfacd542e/A/34f1f60d5fbd41e9a65ea799007f6da7 2024-11-24T22:15:20,351 DEBUG [StoreCloser-TestAcidGuarantees,,1732486488674.cdb3383a4019da5f5ef0568dfacd542e.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/cdb3383a4019da5f5ef0568dfacd542e/A/ca4d693f159f40c89368d00de5a37289 to hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/archive/data/default/TestAcidGuarantees/cdb3383a4019da5f5ef0568dfacd542e/A/ca4d693f159f40c89368d00de5a37289 2024-11-24T22:15:20,352 DEBUG [StoreCloser-TestAcidGuarantees,,1732486488674.cdb3383a4019da5f5ef0568dfacd542e.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/cdb3383a4019da5f5ef0568dfacd542e/A/acf5a3967a5f4ec6b719f46f93df16f7 to hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/archive/data/default/TestAcidGuarantees/cdb3383a4019da5f5ef0568dfacd542e/A/acf5a3967a5f4ec6b719f46f93df16f7 2024-11-24T22:15:20,353 DEBUG [StoreCloser-TestAcidGuarantees,,1732486488674.cdb3383a4019da5f5ef0568dfacd542e.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/cdb3383a4019da5f5ef0568dfacd542e/A/6a5083c0ee774458bf24c31fed15ca5c to hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/archive/data/default/TestAcidGuarantees/cdb3383a4019da5f5ef0568dfacd542e/A/6a5083c0ee774458bf24c31fed15ca5c 2024-11-24T22:15:20,354 DEBUG [StoreCloser-TestAcidGuarantees,,1732486488674.cdb3383a4019da5f5ef0568dfacd542e.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/cdb3383a4019da5f5ef0568dfacd542e/A/79b97ac10a544c658a9ec43f5d94fe7f to hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/archive/data/default/TestAcidGuarantees/cdb3383a4019da5f5ef0568dfacd542e/A/79b97ac10a544c658a9ec43f5d94fe7f 2024-11-24T22:15:20,355 DEBUG [StoreCloser-TestAcidGuarantees,,1732486488674.cdb3383a4019da5f5ef0568dfacd542e.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/cdb3383a4019da5f5ef0568dfacd542e/A/fd12ade02bb949afabf88523cee4fd62 to hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/archive/data/default/TestAcidGuarantees/cdb3383a4019da5f5ef0568dfacd542e/A/fd12ade02bb949afabf88523cee4fd62 2024-11-24T22:15:20,355 DEBUG [StoreCloser-TestAcidGuarantees,,1732486488674.cdb3383a4019da5f5ef0568dfacd542e.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/cdb3383a4019da5f5ef0568dfacd542e/A/e186d6bee8e04f47944534a07e432d52 to hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/archive/data/default/TestAcidGuarantees/cdb3383a4019da5f5ef0568dfacd542e/A/e186d6bee8e04f47944534a07e432d52 2024-11-24T22:15:20,356 DEBUG [StoreCloser-TestAcidGuarantees,,1732486488674.cdb3383a4019da5f5ef0568dfacd542e.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/cdb3383a4019da5f5ef0568dfacd542e/A/22d2943f9286475fbf20ee37ad382e10 to hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/archive/data/default/TestAcidGuarantees/cdb3383a4019da5f5ef0568dfacd542e/A/22d2943f9286475fbf20ee37ad382e10 2024-11-24T22:15:20,357 DEBUG [StoreCloser-TestAcidGuarantees,,1732486488674.cdb3383a4019da5f5ef0568dfacd542e.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/cdb3383a4019da5f5ef0568dfacd542e/A/432138b440f142868293ae834c1ed131 to hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/archive/data/default/TestAcidGuarantees/cdb3383a4019da5f5ef0568dfacd542e/A/432138b440f142868293ae834c1ed131 2024-11-24T22:15:20,358 DEBUG [StoreCloser-TestAcidGuarantees,,1732486488674.cdb3383a4019da5f5ef0568dfacd542e.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/cdb3383a4019da5f5ef0568dfacd542e/A/ff0660ba25d7415a932d59d3791e2c25 to hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/archive/data/default/TestAcidGuarantees/cdb3383a4019da5f5ef0568dfacd542e/A/ff0660ba25d7415a932d59d3791e2c25 2024-11-24T22:15:20,359 DEBUG [StoreCloser-TestAcidGuarantees,,1732486488674.cdb3383a4019da5f5ef0568dfacd542e.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/cdb3383a4019da5f5ef0568dfacd542e/A/9db012e14a6642439d367bf3ad8fe408 to hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/archive/data/default/TestAcidGuarantees/cdb3383a4019da5f5ef0568dfacd542e/A/9db012e14a6642439d367bf3ad8fe408 2024-11-24T22:15:20,360 DEBUG [StoreCloser-TestAcidGuarantees,,1732486488674.cdb3383a4019da5f5ef0568dfacd542e.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/cdb3383a4019da5f5ef0568dfacd542e/A/d5f5c9e33fef41fc8ef8df16567a8fe8 to hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/archive/data/default/TestAcidGuarantees/cdb3383a4019da5f5ef0568dfacd542e/A/d5f5c9e33fef41fc8ef8df16567a8fe8 2024-11-24T22:15:20,361 DEBUG [StoreCloser-TestAcidGuarantees,,1732486488674.cdb3383a4019da5f5ef0568dfacd542e.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/cdb3383a4019da5f5ef0568dfacd542e/A/92e9d617735f4590b5aa7359e2f87d5e to hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/archive/data/default/TestAcidGuarantees/cdb3383a4019da5f5ef0568dfacd542e/A/92e9d617735f4590b5aa7359e2f87d5e 2024-11-24T22:15:20,361 DEBUG [StoreCloser-TestAcidGuarantees,,1732486488674.cdb3383a4019da5f5ef0568dfacd542e.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/cdb3383a4019da5f5ef0568dfacd542e/A/9134c9ad987e430695cb0d7651a28586 to hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/archive/data/default/TestAcidGuarantees/cdb3383a4019da5f5ef0568dfacd542e/A/9134c9ad987e430695cb0d7651a28586 2024-11-24T22:15:20,362 DEBUG [StoreCloser-TestAcidGuarantees,,1732486488674.cdb3383a4019da5f5ef0568dfacd542e.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/cdb3383a4019da5f5ef0568dfacd542e/A/738c0933918f47e68f1f385ea5ede3d9 to hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/archive/data/default/TestAcidGuarantees/cdb3383a4019da5f5ef0568dfacd542e/A/738c0933918f47e68f1f385ea5ede3d9 2024-11-24T22:15:20,363 DEBUG [StoreCloser-TestAcidGuarantees,,1732486488674.cdb3383a4019da5f5ef0568dfacd542e.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/cdb3383a4019da5f5ef0568dfacd542e/A/110c91754e0a45988a2018e87fa8d232 to hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/archive/data/default/TestAcidGuarantees/cdb3383a4019da5f5ef0568dfacd542e/A/110c91754e0a45988a2018e87fa8d232 2024-11-24T22:15:20,364 DEBUG [StoreCloser-TestAcidGuarantees,,1732486488674.cdb3383a4019da5f5ef0568dfacd542e.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/cdb3383a4019da5f5ef0568dfacd542e/A/ea40fb6886784ae982e1a5b7960a5111 to hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/archive/data/default/TestAcidGuarantees/cdb3383a4019da5f5ef0568dfacd542e/A/ea40fb6886784ae982e1a5b7960a5111 2024-11-24T22:15:20,365 DEBUG [StoreCloser-TestAcidGuarantees,,1732486488674.cdb3383a4019da5f5ef0568dfacd542e.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/cdb3383a4019da5f5ef0568dfacd542e/A/7c55c2e8ae634c89bb30793907b35a80 to hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/archive/data/default/TestAcidGuarantees/cdb3383a4019da5f5ef0568dfacd542e/A/7c55c2e8ae634c89bb30793907b35a80 2024-11-24T22:15:20,366 DEBUG [StoreCloser-TestAcidGuarantees,,1732486488674.cdb3383a4019da5f5ef0568dfacd542e.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/cdb3383a4019da5f5ef0568dfacd542e/A/0518f72a50f24932b904aa7048e69b43 to hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/archive/data/default/TestAcidGuarantees/cdb3383a4019da5f5ef0568dfacd542e/A/0518f72a50f24932b904aa7048e69b43 2024-11-24T22:15:20,366 DEBUG [StoreCloser-TestAcidGuarantees,,1732486488674.cdb3383a4019da5f5ef0568dfacd542e.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/cdb3383a4019da5f5ef0568dfacd542e/A/930ac3ca836c49b0a3f4a7c6e9561a75 to hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/archive/data/default/TestAcidGuarantees/cdb3383a4019da5f5ef0568dfacd542e/A/930ac3ca836c49b0a3f4a7c6e9561a75 2024-11-24T22:15:20,367 DEBUG [StoreCloser-TestAcidGuarantees,,1732486488674.cdb3383a4019da5f5ef0568dfacd542e.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/cdb3383a4019da5f5ef0568dfacd542e/A/c60630f768c245218991d2174c071e39 to hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/archive/data/default/TestAcidGuarantees/cdb3383a4019da5f5ef0568dfacd542e/A/c60630f768c245218991d2174c071e39 2024-11-24T22:15:20,369 DEBUG [StoreCloser-TestAcidGuarantees,,1732486488674.cdb3383a4019da5f5ef0568dfacd542e.-1 {}] regionserver.HStore(2317): Moving the files [hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/cdb3383a4019da5f5ef0568dfacd542e/B/aff61553ec6940aa88ba531fe96ccd65, hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/cdb3383a4019da5f5ef0568dfacd542e/B/760c5f08c89a4d1cb6849afadfb67357, hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/cdb3383a4019da5f5ef0568dfacd542e/B/e5b76e9a2c6b4facac53b583784c0fb7, hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/cdb3383a4019da5f5ef0568dfacd542e/B/18ce81563423421c8c6d5c60c19cb405, hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/cdb3383a4019da5f5ef0568dfacd542e/B/3068e6ac5958446782debae8e550dc98, hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/cdb3383a4019da5f5ef0568dfacd542e/B/b68df242e91f4b989faa62390bff6f0d, hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/cdb3383a4019da5f5ef0568dfacd542e/B/c89391f134b744d3b8c1a8421e8cf037, hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/cdb3383a4019da5f5ef0568dfacd542e/B/65d304270e9042d58934bac821362e54, hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/cdb3383a4019da5f5ef0568dfacd542e/B/a2122f90c78d4a8bb8b168164f69b0c9, hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/cdb3383a4019da5f5ef0568dfacd542e/B/c6d4b3ff05d942349fdf2d2a9a078c6c, hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/cdb3383a4019da5f5ef0568dfacd542e/B/2b02afeb3cba495ca56408dafcff1754, hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/cdb3383a4019da5f5ef0568dfacd542e/B/3dc8e173764c4c76ba585c4beebbf519, hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/cdb3383a4019da5f5ef0568dfacd542e/B/99967a88476643f29f948c0d2ad668f0, hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/cdb3383a4019da5f5ef0568dfacd542e/B/e9698a5597854a3380c5735bdb28edab, hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/cdb3383a4019da5f5ef0568dfacd542e/B/e94ce2756c6743089e95d153fcda9749, hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/cdb3383a4019da5f5ef0568dfacd542e/B/67c2d2bcd6e944d1a80d70b92d26f09e, hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/cdb3383a4019da5f5ef0568dfacd542e/B/1511cbf759814b93b4166c8ac54f35ca, hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/cdb3383a4019da5f5ef0568dfacd542e/B/5f46e33e87a64df1ac7d18d27391ff55, hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/cdb3383a4019da5f5ef0568dfacd542e/B/0a697eb0ea8443f0bad97114cc0adf14, hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/cdb3383a4019da5f5ef0568dfacd542e/B/a35577419c784db99f9973e09dcc1e29, hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/cdb3383a4019da5f5ef0568dfacd542e/B/2cd3c91224164040a16965bf25f89bac, hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/cdb3383a4019da5f5ef0568dfacd542e/B/043730d19ec944f5b2bb93ed6b5bfdbd, hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/cdb3383a4019da5f5ef0568dfacd542e/B/93e2d9ce32ef4a27ae31bc67ec3b86a3, hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/cdb3383a4019da5f5ef0568dfacd542e/B/56a31373efde4595be985a3f9afdc70b, hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/cdb3383a4019da5f5ef0568dfacd542e/B/ef3421fddb9a422b990293a78aab8059, hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/cdb3383a4019da5f5ef0568dfacd542e/B/e94895b2f12149d9bfc0802c0a2e7d36, hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/cdb3383a4019da5f5ef0568dfacd542e/B/8508b162a0b54202bd7b981aca6af1c6, hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/cdb3383a4019da5f5ef0568dfacd542e/B/572e0b339dc94be0b36f8ea4a0ef7518, hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/cdb3383a4019da5f5ef0568dfacd542e/B/81e4f86e9c7d41eaa6187447800c6c2f, hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/cdb3383a4019da5f5ef0568dfacd542e/B/2126a18a8f2f46ba8d5ad912a1e84862, hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/cdb3383a4019da5f5ef0568dfacd542e/B/dc4df97aeba4401b8073d9a446f60dd5, hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/cdb3383a4019da5f5ef0568dfacd542e/B/4e906e5b872246cba035629dd76e68f0] to archive 2024-11-24T22:15:20,369 DEBUG [StoreCloser-TestAcidGuarantees,,1732486488674.cdb3383a4019da5f5ef0568dfacd542e.-1 {}] backup.HFileArchiver(360): Archiving compacted files. 2024-11-24T22:15:20,371 DEBUG [StoreCloser-TestAcidGuarantees,,1732486488674.cdb3383a4019da5f5ef0568dfacd542e.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/cdb3383a4019da5f5ef0568dfacd542e/B/aff61553ec6940aa88ba531fe96ccd65 to hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/archive/data/default/TestAcidGuarantees/cdb3383a4019da5f5ef0568dfacd542e/B/aff61553ec6940aa88ba531fe96ccd65 2024-11-24T22:15:20,371 DEBUG [StoreCloser-TestAcidGuarantees,,1732486488674.cdb3383a4019da5f5ef0568dfacd542e.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/cdb3383a4019da5f5ef0568dfacd542e/B/760c5f08c89a4d1cb6849afadfb67357 to hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/archive/data/default/TestAcidGuarantees/cdb3383a4019da5f5ef0568dfacd542e/B/760c5f08c89a4d1cb6849afadfb67357 2024-11-24T22:15:20,372 DEBUG [StoreCloser-TestAcidGuarantees,,1732486488674.cdb3383a4019da5f5ef0568dfacd542e.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/cdb3383a4019da5f5ef0568dfacd542e/B/e5b76e9a2c6b4facac53b583784c0fb7 to hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/archive/data/default/TestAcidGuarantees/cdb3383a4019da5f5ef0568dfacd542e/B/e5b76e9a2c6b4facac53b583784c0fb7 2024-11-24T22:15:20,374 DEBUG [StoreCloser-TestAcidGuarantees,,1732486488674.cdb3383a4019da5f5ef0568dfacd542e.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/cdb3383a4019da5f5ef0568dfacd542e/B/18ce81563423421c8c6d5c60c19cb405 to hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/archive/data/default/TestAcidGuarantees/cdb3383a4019da5f5ef0568dfacd542e/B/18ce81563423421c8c6d5c60c19cb405 2024-11-24T22:15:20,375 DEBUG [StoreCloser-TestAcidGuarantees,,1732486488674.cdb3383a4019da5f5ef0568dfacd542e.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/cdb3383a4019da5f5ef0568dfacd542e/B/3068e6ac5958446782debae8e550dc98 to hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/archive/data/default/TestAcidGuarantees/cdb3383a4019da5f5ef0568dfacd542e/B/3068e6ac5958446782debae8e550dc98 2024-11-24T22:15:20,376 DEBUG [StoreCloser-TestAcidGuarantees,,1732486488674.cdb3383a4019da5f5ef0568dfacd542e.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/cdb3383a4019da5f5ef0568dfacd542e/B/b68df242e91f4b989faa62390bff6f0d to hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/archive/data/default/TestAcidGuarantees/cdb3383a4019da5f5ef0568dfacd542e/B/b68df242e91f4b989faa62390bff6f0d 2024-11-24T22:15:20,377 DEBUG [StoreCloser-TestAcidGuarantees,,1732486488674.cdb3383a4019da5f5ef0568dfacd542e.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/cdb3383a4019da5f5ef0568dfacd542e/B/c89391f134b744d3b8c1a8421e8cf037 to hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/archive/data/default/TestAcidGuarantees/cdb3383a4019da5f5ef0568dfacd542e/B/c89391f134b744d3b8c1a8421e8cf037 2024-11-24T22:15:20,378 DEBUG [StoreCloser-TestAcidGuarantees,,1732486488674.cdb3383a4019da5f5ef0568dfacd542e.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/cdb3383a4019da5f5ef0568dfacd542e/B/65d304270e9042d58934bac821362e54 to hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/archive/data/default/TestAcidGuarantees/cdb3383a4019da5f5ef0568dfacd542e/B/65d304270e9042d58934bac821362e54 2024-11-24T22:15:20,379 DEBUG [StoreCloser-TestAcidGuarantees,,1732486488674.cdb3383a4019da5f5ef0568dfacd542e.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/cdb3383a4019da5f5ef0568dfacd542e/B/a2122f90c78d4a8bb8b168164f69b0c9 to hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/archive/data/default/TestAcidGuarantees/cdb3383a4019da5f5ef0568dfacd542e/B/a2122f90c78d4a8bb8b168164f69b0c9 2024-11-24T22:15:20,380 DEBUG [StoreCloser-TestAcidGuarantees,,1732486488674.cdb3383a4019da5f5ef0568dfacd542e.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/cdb3383a4019da5f5ef0568dfacd542e/B/c6d4b3ff05d942349fdf2d2a9a078c6c to hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/archive/data/default/TestAcidGuarantees/cdb3383a4019da5f5ef0568dfacd542e/B/c6d4b3ff05d942349fdf2d2a9a078c6c 2024-11-24T22:15:20,381 DEBUG [StoreCloser-TestAcidGuarantees,,1732486488674.cdb3383a4019da5f5ef0568dfacd542e.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/cdb3383a4019da5f5ef0568dfacd542e/B/2b02afeb3cba495ca56408dafcff1754 to hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/archive/data/default/TestAcidGuarantees/cdb3383a4019da5f5ef0568dfacd542e/B/2b02afeb3cba495ca56408dafcff1754 2024-11-24T22:15:20,381 DEBUG [StoreCloser-TestAcidGuarantees,,1732486488674.cdb3383a4019da5f5ef0568dfacd542e.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/cdb3383a4019da5f5ef0568dfacd542e/B/3dc8e173764c4c76ba585c4beebbf519 to hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/archive/data/default/TestAcidGuarantees/cdb3383a4019da5f5ef0568dfacd542e/B/3dc8e173764c4c76ba585c4beebbf519 2024-11-24T22:15:20,382 DEBUG [StoreCloser-TestAcidGuarantees,,1732486488674.cdb3383a4019da5f5ef0568dfacd542e.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/cdb3383a4019da5f5ef0568dfacd542e/B/99967a88476643f29f948c0d2ad668f0 to hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/archive/data/default/TestAcidGuarantees/cdb3383a4019da5f5ef0568dfacd542e/B/99967a88476643f29f948c0d2ad668f0 2024-11-24T22:15:20,383 DEBUG [StoreCloser-TestAcidGuarantees,,1732486488674.cdb3383a4019da5f5ef0568dfacd542e.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/cdb3383a4019da5f5ef0568dfacd542e/B/e9698a5597854a3380c5735bdb28edab to hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/archive/data/default/TestAcidGuarantees/cdb3383a4019da5f5ef0568dfacd542e/B/e9698a5597854a3380c5735bdb28edab 2024-11-24T22:15:20,384 DEBUG [StoreCloser-TestAcidGuarantees,,1732486488674.cdb3383a4019da5f5ef0568dfacd542e.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/cdb3383a4019da5f5ef0568dfacd542e/B/e94ce2756c6743089e95d153fcda9749 to hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/archive/data/default/TestAcidGuarantees/cdb3383a4019da5f5ef0568dfacd542e/B/e94ce2756c6743089e95d153fcda9749 2024-11-24T22:15:20,385 DEBUG [StoreCloser-TestAcidGuarantees,,1732486488674.cdb3383a4019da5f5ef0568dfacd542e.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/cdb3383a4019da5f5ef0568dfacd542e/B/67c2d2bcd6e944d1a80d70b92d26f09e to hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/archive/data/default/TestAcidGuarantees/cdb3383a4019da5f5ef0568dfacd542e/B/67c2d2bcd6e944d1a80d70b92d26f09e 2024-11-24T22:15:20,386 DEBUG [StoreCloser-TestAcidGuarantees,,1732486488674.cdb3383a4019da5f5ef0568dfacd542e.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/cdb3383a4019da5f5ef0568dfacd542e/B/1511cbf759814b93b4166c8ac54f35ca to hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/archive/data/default/TestAcidGuarantees/cdb3383a4019da5f5ef0568dfacd542e/B/1511cbf759814b93b4166c8ac54f35ca 2024-11-24T22:15:20,387 DEBUG [StoreCloser-TestAcidGuarantees,,1732486488674.cdb3383a4019da5f5ef0568dfacd542e.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/cdb3383a4019da5f5ef0568dfacd542e/B/5f46e33e87a64df1ac7d18d27391ff55 to hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/archive/data/default/TestAcidGuarantees/cdb3383a4019da5f5ef0568dfacd542e/B/5f46e33e87a64df1ac7d18d27391ff55 2024-11-24T22:15:20,388 DEBUG [StoreCloser-TestAcidGuarantees,,1732486488674.cdb3383a4019da5f5ef0568dfacd542e.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/cdb3383a4019da5f5ef0568dfacd542e/B/0a697eb0ea8443f0bad97114cc0adf14 to hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/archive/data/default/TestAcidGuarantees/cdb3383a4019da5f5ef0568dfacd542e/B/0a697eb0ea8443f0bad97114cc0adf14 2024-11-24T22:15:20,389 DEBUG [StoreCloser-TestAcidGuarantees,,1732486488674.cdb3383a4019da5f5ef0568dfacd542e.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/cdb3383a4019da5f5ef0568dfacd542e/B/a35577419c784db99f9973e09dcc1e29 to hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/archive/data/default/TestAcidGuarantees/cdb3383a4019da5f5ef0568dfacd542e/B/a35577419c784db99f9973e09dcc1e29 2024-11-24T22:15:20,390 DEBUG [StoreCloser-TestAcidGuarantees,,1732486488674.cdb3383a4019da5f5ef0568dfacd542e.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/cdb3383a4019da5f5ef0568dfacd542e/B/2cd3c91224164040a16965bf25f89bac to hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/archive/data/default/TestAcidGuarantees/cdb3383a4019da5f5ef0568dfacd542e/B/2cd3c91224164040a16965bf25f89bac 2024-11-24T22:15:20,390 DEBUG [StoreCloser-TestAcidGuarantees,,1732486488674.cdb3383a4019da5f5ef0568dfacd542e.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/cdb3383a4019da5f5ef0568dfacd542e/B/043730d19ec944f5b2bb93ed6b5bfdbd to hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/archive/data/default/TestAcidGuarantees/cdb3383a4019da5f5ef0568dfacd542e/B/043730d19ec944f5b2bb93ed6b5bfdbd 2024-11-24T22:15:20,391 DEBUG [StoreCloser-TestAcidGuarantees,,1732486488674.cdb3383a4019da5f5ef0568dfacd542e.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/cdb3383a4019da5f5ef0568dfacd542e/B/93e2d9ce32ef4a27ae31bc67ec3b86a3 to hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/archive/data/default/TestAcidGuarantees/cdb3383a4019da5f5ef0568dfacd542e/B/93e2d9ce32ef4a27ae31bc67ec3b86a3 2024-11-24T22:15:20,392 DEBUG [StoreCloser-TestAcidGuarantees,,1732486488674.cdb3383a4019da5f5ef0568dfacd542e.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/cdb3383a4019da5f5ef0568dfacd542e/B/56a31373efde4595be985a3f9afdc70b to hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/archive/data/default/TestAcidGuarantees/cdb3383a4019da5f5ef0568dfacd542e/B/56a31373efde4595be985a3f9afdc70b 2024-11-24T22:15:20,393 DEBUG [StoreCloser-TestAcidGuarantees,,1732486488674.cdb3383a4019da5f5ef0568dfacd542e.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/cdb3383a4019da5f5ef0568dfacd542e/B/ef3421fddb9a422b990293a78aab8059 to hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/archive/data/default/TestAcidGuarantees/cdb3383a4019da5f5ef0568dfacd542e/B/ef3421fddb9a422b990293a78aab8059 2024-11-24T22:15:20,394 DEBUG [StoreCloser-TestAcidGuarantees,,1732486488674.cdb3383a4019da5f5ef0568dfacd542e.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/cdb3383a4019da5f5ef0568dfacd542e/B/e94895b2f12149d9bfc0802c0a2e7d36 to hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/archive/data/default/TestAcidGuarantees/cdb3383a4019da5f5ef0568dfacd542e/B/e94895b2f12149d9bfc0802c0a2e7d36 2024-11-24T22:15:20,395 DEBUG [StoreCloser-TestAcidGuarantees,,1732486488674.cdb3383a4019da5f5ef0568dfacd542e.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/cdb3383a4019da5f5ef0568dfacd542e/B/8508b162a0b54202bd7b981aca6af1c6 to hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/archive/data/default/TestAcidGuarantees/cdb3383a4019da5f5ef0568dfacd542e/B/8508b162a0b54202bd7b981aca6af1c6 2024-11-24T22:15:20,396 DEBUG [StoreCloser-TestAcidGuarantees,,1732486488674.cdb3383a4019da5f5ef0568dfacd542e.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/cdb3383a4019da5f5ef0568dfacd542e/B/572e0b339dc94be0b36f8ea4a0ef7518 to hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/archive/data/default/TestAcidGuarantees/cdb3383a4019da5f5ef0568dfacd542e/B/572e0b339dc94be0b36f8ea4a0ef7518 2024-11-24T22:15:20,397 DEBUG [StoreCloser-TestAcidGuarantees,,1732486488674.cdb3383a4019da5f5ef0568dfacd542e.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/cdb3383a4019da5f5ef0568dfacd542e/B/81e4f86e9c7d41eaa6187447800c6c2f to hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/archive/data/default/TestAcidGuarantees/cdb3383a4019da5f5ef0568dfacd542e/B/81e4f86e9c7d41eaa6187447800c6c2f 2024-11-24T22:15:20,397 DEBUG [StoreCloser-TestAcidGuarantees,,1732486488674.cdb3383a4019da5f5ef0568dfacd542e.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/cdb3383a4019da5f5ef0568dfacd542e/B/2126a18a8f2f46ba8d5ad912a1e84862 to hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/archive/data/default/TestAcidGuarantees/cdb3383a4019da5f5ef0568dfacd542e/B/2126a18a8f2f46ba8d5ad912a1e84862 2024-11-24T22:15:20,398 DEBUG [StoreCloser-TestAcidGuarantees,,1732486488674.cdb3383a4019da5f5ef0568dfacd542e.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/cdb3383a4019da5f5ef0568dfacd542e/B/dc4df97aeba4401b8073d9a446f60dd5 to hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/archive/data/default/TestAcidGuarantees/cdb3383a4019da5f5ef0568dfacd542e/B/dc4df97aeba4401b8073d9a446f60dd5 2024-11-24T22:15:20,399 DEBUG [StoreCloser-TestAcidGuarantees,,1732486488674.cdb3383a4019da5f5ef0568dfacd542e.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/cdb3383a4019da5f5ef0568dfacd542e/B/4e906e5b872246cba035629dd76e68f0 to hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/archive/data/default/TestAcidGuarantees/cdb3383a4019da5f5ef0568dfacd542e/B/4e906e5b872246cba035629dd76e68f0 2024-11-24T22:15:20,400 DEBUG [StoreCloser-TestAcidGuarantees,,1732486488674.cdb3383a4019da5f5ef0568dfacd542e.-1 {}] regionserver.HStore(2317): Moving the files [hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/cdb3383a4019da5f5ef0568dfacd542e/C/9af272d332df4b0792e8c07b7f46a69f, hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/cdb3383a4019da5f5ef0568dfacd542e/C/3a37abeb7cea46f495ee0f637ba2b9fb, hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/cdb3383a4019da5f5ef0568dfacd542e/C/73ebb44821fc486aa70667b86beeabc6, hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/cdb3383a4019da5f5ef0568dfacd542e/C/94f82c5bbcdd4208abaa8b61d53da1cb, hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/cdb3383a4019da5f5ef0568dfacd542e/C/54172941587641b39b3076d159e66605, hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/cdb3383a4019da5f5ef0568dfacd542e/C/48dcc83ad5644812ba027d775c3a45f7, hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/cdb3383a4019da5f5ef0568dfacd542e/C/3b88cd9780714be0ba6d5f570030a441, hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/cdb3383a4019da5f5ef0568dfacd542e/C/42c1b6cf23734fef8a3529416a6735e3, hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/cdb3383a4019da5f5ef0568dfacd542e/C/b73271238a90407aae9295d57f98608a, hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/cdb3383a4019da5f5ef0568dfacd542e/C/112218e9c5d848ba91829de0208e34b2, hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/cdb3383a4019da5f5ef0568dfacd542e/C/062cecfaead74cc6864f360d41e8e99f, hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/cdb3383a4019da5f5ef0568dfacd542e/C/a9bbb3f3509d406a976860c0c208d7c3, hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/cdb3383a4019da5f5ef0568dfacd542e/C/6408a0d1112d4fa9853e63edbdd01c09, hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/cdb3383a4019da5f5ef0568dfacd542e/C/cf79c39e11294570bfd18dc6ef2c4108, hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/cdb3383a4019da5f5ef0568dfacd542e/C/6ad8c8648bc049cdb645eb8af4e16679, hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/cdb3383a4019da5f5ef0568dfacd542e/C/c112743a0d3747e89caee36a9f12cf6f, hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/cdb3383a4019da5f5ef0568dfacd542e/C/ab7e06f8486a4a7583de468afccbd8b7, hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/cdb3383a4019da5f5ef0568dfacd542e/C/4a20a25ec25d4e2ab5355cba35fbd701, hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/cdb3383a4019da5f5ef0568dfacd542e/C/541524ca4f294711b5cba8c749f4fcd5, hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/cdb3383a4019da5f5ef0568dfacd542e/C/59837082e57443849300f8992554f804, hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/cdb3383a4019da5f5ef0568dfacd542e/C/e8dc99fbf56546d6a0a6358cb5dbd3d0, hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/cdb3383a4019da5f5ef0568dfacd542e/C/58c7394de6c14021aae54876cf65fa09, hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/cdb3383a4019da5f5ef0568dfacd542e/C/485cf734e750420f950cac587454942a, hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/cdb3383a4019da5f5ef0568dfacd542e/C/2ea8b80df3ec498ea0b9fa5886e9194a, hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/cdb3383a4019da5f5ef0568dfacd542e/C/0defd1196d7f48eab2401586ed1b976d, hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/cdb3383a4019da5f5ef0568dfacd542e/C/f4856e4fded1464d906bb6d736463799, hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/cdb3383a4019da5f5ef0568dfacd542e/C/441c2523681548fa9d09028a3484fa21, hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/cdb3383a4019da5f5ef0568dfacd542e/C/845e21e625844395ad910b7963d0e3fc, hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/cdb3383a4019da5f5ef0568dfacd542e/C/e522bb98c69848cf83b98d352440eb70, hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/cdb3383a4019da5f5ef0568dfacd542e/C/fe3bc641923a4531b63d749f303ae574, hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/cdb3383a4019da5f5ef0568dfacd542e/C/2bb428b726714735a7a3359a0464ba12, hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/cdb3383a4019da5f5ef0568dfacd542e/C/9dcfc731809446bdbe1c35518b60caee] to archive 2024-11-24T22:15:20,401 DEBUG [StoreCloser-TestAcidGuarantees,,1732486488674.cdb3383a4019da5f5ef0568dfacd542e.-1 {}] backup.HFileArchiver(360): Archiving compacted files. 2024-11-24T22:15:20,402 DEBUG [StoreCloser-TestAcidGuarantees,,1732486488674.cdb3383a4019da5f5ef0568dfacd542e.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/cdb3383a4019da5f5ef0568dfacd542e/C/9af272d332df4b0792e8c07b7f46a69f to hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/archive/data/default/TestAcidGuarantees/cdb3383a4019da5f5ef0568dfacd542e/C/9af272d332df4b0792e8c07b7f46a69f 2024-11-24T22:15:20,403 DEBUG [StoreCloser-TestAcidGuarantees,,1732486488674.cdb3383a4019da5f5ef0568dfacd542e.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/cdb3383a4019da5f5ef0568dfacd542e/C/3a37abeb7cea46f495ee0f637ba2b9fb to hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/archive/data/default/TestAcidGuarantees/cdb3383a4019da5f5ef0568dfacd542e/C/3a37abeb7cea46f495ee0f637ba2b9fb 2024-11-24T22:15:20,404 DEBUG [StoreCloser-TestAcidGuarantees,,1732486488674.cdb3383a4019da5f5ef0568dfacd542e.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/cdb3383a4019da5f5ef0568dfacd542e/C/73ebb44821fc486aa70667b86beeabc6 to hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/archive/data/default/TestAcidGuarantees/cdb3383a4019da5f5ef0568dfacd542e/C/73ebb44821fc486aa70667b86beeabc6 2024-11-24T22:15:20,405 DEBUG [StoreCloser-TestAcidGuarantees,,1732486488674.cdb3383a4019da5f5ef0568dfacd542e.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/cdb3383a4019da5f5ef0568dfacd542e/C/94f82c5bbcdd4208abaa8b61d53da1cb to hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/archive/data/default/TestAcidGuarantees/cdb3383a4019da5f5ef0568dfacd542e/C/94f82c5bbcdd4208abaa8b61d53da1cb 2024-11-24T22:15:20,406 DEBUG [StoreCloser-TestAcidGuarantees,,1732486488674.cdb3383a4019da5f5ef0568dfacd542e.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/cdb3383a4019da5f5ef0568dfacd542e/C/54172941587641b39b3076d159e66605 to hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/archive/data/default/TestAcidGuarantees/cdb3383a4019da5f5ef0568dfacd542e/C/54172941587641b39b3076d159e66605 2024-11-24T22:15:20,407 DEBUG [StoreCloser-TestAcidGuarantees,,1732486488674.cdb3383a4019da5f5ef0568dfacd542e.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/cdb3383a4019da5f5ef0568dfacd542e/C/48dcc83ad5644812ba027d775c3a45f7 to hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/archive/data/default/TestAcidGuarantees/cdb3383a4019da5f5ef0568dfacd542e/C/48dcc83ad5644812ba027d775c3a45f7 2024-11-24T22:15:20,408 DEBUG [StoreCloser-TestAcidGuarantees,,1732486488674.cdb3383a4019da5f5ef0568dfacd542e.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/cdb3383a4019da5f5ef0568dfacd542e/C/3b88cd9780714be0ba6d5f570030a441 to hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/archive/data/default/TestAcidGuarantees/cdb3383a4019da5f5ef0568dfacd542e/C/3b88cd9780714be0ba6d5f570030a441 2024-11-24T22:15:20,409 DEBUG [StoreCloser-TestAcidGuarantees,,1732486488674.cdb3383a4019da5f5ef0568dfacd542e.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/cdb3383a4019da5f5ef0568dfacd542e/C/42c1b6cf23734fef8a3529416a6735e3 to hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/archive/data/default/TestAcidGuarantees/cdb3383a4019da5f5ef0568dfacd542e/C/42c1b6cf23734fef8a3529416a6735e3 2024-11-24T22:15:20,410 DEBUG [StoreCloser-TestAcidGuarantees,,1732486488674.cdb3383a4019da5f5ef0568dfacd542e.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/cdb3383a4019da5f5ef0568dfacd542e/C/b73271238a90407aae9295d57f98608a to hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/archive/data/default/TestAcidGuarantees/cdb3383a4019da5f5ef0568dfacd542e/C/b73271238a90407aae9295d57f98608a 2024-11-24T22:15:20,411 DEBUG [StoreCloser-TestAcidGuarantees,,1732486488674.cdb3383a4019da5f5ef0568dfacd542e.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/cdb3383a4019da5f5ef0568dfacd542e/C/112218e9c5d848ba91829de0208e34b2 to hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/archive/data/default/TestAcidGuarantees/cdb3383a4019da5f5ef0568dfacd542e/C/112218e9c5d848ba91829de0208e34b2 2024-11-24T22:15:20,412 DEBUG [StoreCloser-TestAcidGuarantees,,1732486488674.cdb3383a4019da5f5ef0568dfacd542e.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/cdb3383a4019da5f5ef0568dfacd542e/C/062cecfaead74cc6864f360d41e8e99f to hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/archive/data/default/TestAcidGuarantees/cdb3383a4019da5f5ef0568dfacd542e/C/062cecfaead74cc6864f360d41e8e99f 2024-11-24T22:15:20,413 DEBUG [StoreCloser-TestAcidGuarantees,,1732486488674.cdb3383a4019da5f5ef0568dfacd542e.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/cdb3383a4019da5f5ef0568dfacd542e/C/a9bbb3f3509d406a976860c0c208d7c3 to hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/archive/data/default/TestAcidGuarantees/cdb3383a4019da5f5ef0568dfacd542e/C/a9bbb3f3509d406a976860c0c208d7c3 2024-11-24T22:15:20,414 DEBUG [StoreCloser-TestAcidGuarantees,,1732486488674.cdb3383a4019da5f5ef0568dfacd542e.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/cdb3383a4019da5f5ef0568dfacd542e/C/6408a0d1112d4fa9853e63edbdd01c09 to hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/archive/data/default/TestAcidGuarantees/cdb3383a4019da5f5ef0568dfacd542e/C/6408a0d1112d4fa9853e63edbdd01c09 2024-11-24T22:15:20,415 DEBUG [StoreCloser-TestAcidGuarantees,,1732486488674.cdb3383a4019da5f5ef0568dfacd542e.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/cdb3383a4019da5f5ef0568dfacd542e/C/cf79c39e11294570bfd18dc6ef2c4108 to hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/archive/data/default/TestAcidGuarantees/cdb3383a4019da5f5ef0568dfacd542e/C/cf79c39e11294570bfd18dc6ef2c4108 2024-11-24T22:15:20,416 DEBUG [StoreCloser-TestAcidGuarantees,,1732486488674.cdb3383a4019da5f5ef0568dfacd542e.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/cdb3383a4019da5f5ef0568dfacd542e/C/6ad8c8648bc049cdb645eb8af4e16679 to hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/archive/data/default/TestAcidGuarantees/cdb3383a4019da5f5ef0568dfacd542e/C/6ad8c8648bc049cdb645eb8af4e16679 2024-11-24T22:15:20,417 DEBUG [StoreCloser-TestAcidGuarantees,,1732486488674.cdb3383a4019da5f5ef0568dfacd542e.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/cdb3383a4019da5f5ef0568dfacd542e/C/c112743a0d3747e89caee36a9f12cf6f to hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/archive/data/default/TestAcidGuarantees/cdb3383a4019da5f5ef0568dfacd542e/C/c112743a0d3747e89caee36a9f12cf6f 2024-11-24T22:15:20,418 DEBUG [StoreCloser-TestAcidGuarantees,,1732486488674.cdb3383a4019da5f5ef0568dfacd542e.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/cdb3383a4019da5f5ef0568dfacd542e/C/ab7e06f8486a4a7583de468afccbd8b7 to hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/archive/data/default/TestAcidGuarantees/cdb3383a4019da5f5ef0568dfacd542e/C/ab7e06f8486a4a7583de468afccbd8b7 2024-11-24T22:15:20,418 DEBUG [StoreCloser-TestAcidGuarantees,,1732486488674.cdb3383a4019da5f5ef0568dfacd542e.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/cdb3383a4019da5f5ef0568dfacd542e/C/4a20a25ec25d4e2ab5355cba35fbd701 to hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/archive/data/default/TestAcidGuarantees/cdb3383a4019da5f5ef0568dfacd542e/C/4a20a25ec25d4e2ab5355cba35fbd701 2024-11-24T22:15:20,419 DEBUG [StoreCloser-TestAcidGuarantees,,1732486488674.cdb3383a4019da5f5ef0568dfacd542e.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/cdb3383a4019da5f5ef0568dfacd542e/C/541524ca4f294711b5cba8c749f4fcd5 to hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/archive/data/default/TestAcidGuarantees/cdb3383a4019da5f5ef0568dfacd542e/C/541524ca4f294711b5cba8c749f4fcd5 2024-11-24T22:15:20,420 DEBUG [StoreCloser-TestAcidGuarantees,,1732486488674.cdb3383a4019da5f5ef0568dfacd542e.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/cdb3383a4019da5f5ef0568dfacd542e/C/59837082e57443849300f8992554f804 to hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/archive/data/default/TestAcidGuarantees/cdb3383a4019da5f5ef0568dfacd542e/C/59837082e57443849300f8992554f804 2024-11-24T22:15:20,421 DEBUG [StoreCloser-TestAcidGuarantees,,1732486488674.cdb3383a4019da5f5ef0568dfacd542e.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/cdb3383a4019da5f5ef0568dfacd542e/C/e8dc99fbf56546d6a0a6358cb5dbd3d0 to hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/archive/data/default/TestAcidGuarantees/cdb3383a4019da5f5ef0568dfacd542e/C/e8dc99fbf56546d6a0a6358cb5dbd3d0 2024-11-24T22:15:20,422 DEBUG [StoreCloser-TestAcidGuarantees,,1732486488674.cdb3383a4019da5f5ef0568dfacd542e.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/cdb3383a4019da5f5ef0568dfacd542e/C/58c7394de6c14021aae54876cf65fa09 to hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/archive/data/default/TestAcidGuarantees/cdb3383a4019da5f5ef0568dfacd542e/C/58c7394de6c14021aae54876cf65fa09 2024-11-24T22:15:20,423 DEBUG [StoreCloser-TestAcidGuarantees,,1732486488674.cdb3383a4019da5f5ef0568dfacd542e.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/cdb3383a4019da5f5ef0568dfacd542e/C/485cf734e750420f950cac587454942a to hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/archive/data/default/TestAcidGuarantees/cdb3383a4019da5f5ef0568dfacd542e/C/485cf734e750420f950cac587454942a 2024-11-24T22:15:20,423 DEBUG [StoreCloser-TestAcidGuarantees,,1732486488674.cdb3383a4019da5f5ef0568dfacd542e.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/cdb3383a4019da5f5ef0568dfacd542e/C/2ea8b80df3ec498ea0b9fa5886e9194a to hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/archive/data/default/TestAcidGuarantees/cdb3383a4019da5f5ef0568dfacd542e/C/2ea8b80df3ec498ea0b9fa5886e9194a 2024-11-24T22:15:20,424 DEBUG [StoreCloser-TestAcidGuarantees,,1732486488674.cdb3383a4019da5f5ef0568dfacd542e.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/cdb3383a4019da5f5ef0568dfacd542e/C/0defd1196d7f48eab2401586ed1b976d to hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/archive/data/default/TestAcidGuarantees/cdb3383a4019da5f5ef0568dfacd542e/C/0defd1196d7f48eab2401586ed1b976d 2024-11-24T22:15:20,425 DEBUG [StoreCloser-TestAcidGuarantees,,1732486488674.cdb3383a4019da5f5ef0568dfacd542e.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/cdb3383a4019da5f5ef0568dfacd542e/C/f4856e4fded1464d906bb6d736463799 to hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/archive/data/default/TestAcidGuarantees/cdb3383a4019da5f5ef0568dfacd542e/C/f4856e4fded1464d906bb6d736463799 2024-11-24T22:15:20,426 DEBUG [StoreCloser-TestAcidGuarantees,,1732486488674.cdb3383a4019da5f5ef0568dfacd542e.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/cdb3383a4019da5f5ef0568dfacd542e/C/441c2523681548fa9d09028a3484fa21 to hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/archive/data/default/TestAcidGuarantees/cdb3383a4019da5f5ef0568dfacd542e/C/441c2523681548fa9d09028a3484fa21 2024-11-24T22:15:20,427 DEBUG [StoreCloser-TestAcidGuarantees,,1732486488674.cdb3383a4019da5f5ef0568dfacd542e.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/cdb3383a4019da5f5ef0568dfacd542e/C/845e21e625844395ad910b7963d0e3fc to hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/archive/data/default/TestAcidGuarantees/cdb3383a4019da5f5ef0568dfacd542e/C/845e21e625844395ad910b7963d0e3fc 2024-11-24T22:15:20,428 DEBUG [StoreCloser-TestAcidGuarantees,,1732486488674.cdb3383a4019da5f5ef0568dfacd542e.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/cdb3383a4019da5f5ef0568dfacd542e/C/e522bb98c69848cf83b98d352440eb70 to hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/archive/data/default/TestAcidGuarantees/cdb3383a4019da5f5ef0568dfacd542e/C/e522bb98c69848cf83b98d352440eb70 2024-11-24T22:15:20,429 DEBUG [StoreCloser-TestAcidGuarantees,,1732486488674.cdb3383a4019da5f5ef0568dfacd542e.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/cdb3383a4019da5f5ef0568dfacd542e/C/fe3bc641923a4531b63d749f303ae574 to hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/archive/data/default/TestAcidGuarantees/cdb3383a4019da5f5ef0568dfacd542e/C/fe3bc641923a4531b63d749f303ae574 2024-11-24T22:15:20,430 DEBUG [StoreCloser-TestAcidGuarantees,,1732486488674.cdb3383a4019da5f5ef0568dfacd542e.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/cdb3383a4019da5f5ef0568dfacd542e/C/2bb428b726714735a7a3359a0464ba12 to hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/archive/data/default/TestAcidGuarantees/cdb3383a4019da5f5ef0568dfacd542e/C/2bb428b726714735a7a3359a0464ba12 2024-11-24T22:15:20,431 DEBUG [StoreCloser-TestAcidGuarantees,,1732486488674.cdb3383a4019da5f5ef0568dfacd542e.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/cdb3383a4019da5f5ef0568dfacd542e/C/9dcfc731809446bdbe1c35518b60caee to hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/archive/data/default/TestAcidGuarantees/cdb3383a4019da5f5ef0568dfacd542e/C/9dcfc731809446bdbe1c35518b60caee 2024-11-24T22:15:20,440 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37701 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=84 2024-11-24T22:15:20,442 DEBUG [RS_CLOSE_REGION-regionserver/bd6592c37bb2:0-0 {event_type=M_RS_CLOSE_REGION, pid=87}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/cdb3383a4019da5f5ef0568dfacd542e/recovered.edits/498.seqid, newMaxSeqId=498, maxSeqId=1 2024-11-24T22:15:20,443 INFO [RS_CLOSE_REGION-regionserver/bd6592c37bb2:0-0 {event_type=M_RS_CLOSE_REGION, pid=87}] regionserver.HRegion(1973): Closed TestAcidGuarantees,,1732486488674.cdb3383a4019da5f5ef0568dfacd542e. 2024-11-24T22:15:20,443 DEBUG [RS_CLOSE_REGION-regionserver/bd6592c37bb2:0-0 {event_type=M_RS_CLOSE_REGION, pid=87}] regionserver.HRegion(1676): Region close journal for cdb3383a4019da5f5ef0568dfacd542e: Waiting for close lock at 1732486519464Running coprocessor pre-close hooks at 1732486519464Disabling compacts and flushes for region at 1732486519464Disabling writes for close at 1732486519465 (+1 ms)Obtaining lock to block concurrent updates at 1732486519465Preparing flush snapshotting stores in cdb3383a4019da5f5ef0568dfacd542e at 1732486519465Finished memstore snapshotting TestAcidGuarantees,,1732486488674.cdb3383a4019da5f5ef0568dfacd542e., syncing WAL and waiting on mvcc, flushsize=dataSize=34350, getHeapSize=102720, getOffHeapSize=0, getCellsCount=750 at 1732486519465Flushing stores of TestAcidGuarantees,,1732486488674.cdb3383a4019da5f5ef0568dfacd542e. at 1732486519465Flushing cdb3383a4019da5f5ef0568dfacd542e/A: creating writer at 1732486519466 (+1 ms)Flushing cdb3383a4019da5f5ef0568dfacd542e/A: appending metadata at 1732486519468 (+2 ms)Flushing cdb3383a4019da5f5ef0568dfacd542e/A: closing flushed file at 1732486519468Flushing cdb3383a4019da5f5ef0568dfacd542e/B: creating writer at 1732486519879 (+411 ms)Flushing cdb3383a4019da5f5ef0568dfacd542e/B: appending metadata at 1732486519882 (+3 ms)Flushing cdb3383a4019da5f5ef0568dfacd542e/B: closing flushed file at 1732486519882Flushing cdb3383a4019da5f5ef0568dfacd542e/C: creating writer at 1732486519907 (+25 ms)Flushing cdb3383a4019da5f5ef0568dfacd542e/C: appending metadata at 1732486519911 (+4 ms)Flushing cdb3383a4019da5f5ef0568dfacd542e/C: closing flushed file at 1732486519911Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@1bf2f8a1: reopening flushed file at 1732486520324 (+413 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@4950c3d9: reopening flushed file at 1732486520329 (+5 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@4ef13e01: reopening flushed file at 1732486520333 (+4 ms)Finished flush of dataSize ~33.54 KB/34350, heapSize ~100.31 KB/102720, currentSize=0 B/0 for cdb3383a4019da5f5ef0568dfacd542e in 873ms, sequenceid=495, compaction requested=true at 1732486520338 (+5 ms)Writing region close event to WAL at 1732486520432 (+94 ms)Running coprocessor post-close hooks at 1732486520443 (+11 ms)Closed at 1732486520443 2024-11-24T22:15:20,445 INFO [RS_CLOSE_REGION-regionserver/bd6592c37bb2:0-0 {event_type=M_RS_CLOSE_REGION, pid=87}] handler.UnassignRegionHandler(157): Closed cdb3383a4019da5f5ef0568dfacd542e 2024-11-24T22:15:20,445 INFO [PEWorker-3 {}] assignment.RegionStateStore(223): pid=86 updating hbase:meta row=cdb3383a4019da5f5ef0568dfacd542e, regionState=CLOSED 2024-11-24T22:15:20,447 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=87, ppid=86, state=RUNNABLE, hasLock=false; CloseRegionProcedure cdb3383a4019da5f5ef0568dfacd542e, server=bd6592c37bb2,40087,1732486423713 because future has completed 2024-11-24T22:15:20,449 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=87, resume processing ppid=86 2024-11-24T22:15:20,449 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=87, ppid=86, state=SUCCESS, hasLock=false; CloseRegionProcedure cdb3383a4019da5f5ef0568dfacd542e, server=bd6592c37bb2,40087,1732486423713 in 1.1360 sec 2024-11-24T22:15:20,450 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=86, resume processing ppid=85 2024-11-24T22:15:20,451 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=86, ppid=85, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=cdb3383a4019da5f5ef0568dfacd542e, UNASSIGN in 1.1400 sec 2024-11-24T22:15:20,452 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=85, resume processing ppid=84 2024-11-24T22:15:20,452 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=85, ppid=84, state=SUCCESS, hasLock=false; CloseTableRegionsProcedure table=TestAcidGuarantees in 1.1430 sec 2024-11-24T22:15:20,454 DEBUG [PEWorker-5 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1732486520453"}]},"ts":"1732486520453"} 2024-11-24T22:15:20,455 INFO [PEWorker-5 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=DISABLED in hbase:meta 2024-11-24T22:15:20,455 INFO [PEWorker-5 {}] procedure.DisableTableProcedure(296): Set TestAcidGuarantees to state=DISABLED 2024-11-24T22:15:20,457 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=84, state=SUCCESS, hasLock=false; DisableTableProcedure table=TestAcidGuarantees in 1.1510 sec 2024-11-24T22:15:21,451 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37701 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=84 2024-11-24T22:15:21,451 INFO [RPCClient-NioEventLoopGroup-4-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: DISABLE, Table Name: default:TestAcidGuarantees completed 2024-11-24T22:15:21,451 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37701 {}] master.HMaster$5(2570): Client=jenkins//172.17.0.2 delete TestAcidGuarantees 2024-11-24T22:15:21,452 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37701 {}] procedure2.ProcedureExecutor(1139): Stored pid=88, state=RUNNABLE:DELETE_TABLE_PRE_OPERATION, hasLock=false; DeleteTableProcedure table=TestAcidGuarantees 2024-11-24T22:15:21,453 DEBUG [PEWorker-3 {}] procedure.DeleteTableProcedure(103): Waiting for RIT for pid=88, state=RUNNABLE:DELETE_TABLE_PRE_OPERATION, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-24T22:15:21,454 DEBUG [PEWorker-3 {}] procedure.DeleteTableProcedure(115): Deleting regions from filesystem for pid=88, state=RUNNABLE:DELETE_TABLE_CLEAR_FS_LAYOUT, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-24T22:15:21,454 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37701 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=88 2024-11-24T22:15:21,456 DEBUG [HFileArchiver-3 {}] backup.HFileArchiver(131): ARCHIVING hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/cdb3383a4019da5f5ef0568dfacd542e 2024-11-24T22:15:21,458 DEBUG [HFileArchiver-3 {}] backup.HFileArchiver(159): Archiving [FileablePath, hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/cdb3383a4019da5f5ef0568dfacd542e/A, FileablePath, hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/cdb3383a4019da5f5ef0568dfacd542e/B, FileablePath, hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/cdb3383a4019da5f5ef0568dfacd542e/C, FileablePath, hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/cdb3383a4019da5f5ef0568dfacd542e/recovered.edits] 2024-11-24T22:15:21,462 DEBUG [HFileArchiver-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/cdb3383a4019da5f5ef0568dfacd542e/A/5dc71dd99901496ca0d627d3c126042a to hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/archive/data/default/TestAcidGuarantees/cdb3383a4019da5f5ef0568dfacd542e/A/5dc71dd99901496ca0d627d3c126042a 2024-11-24T22:15:21,463 DEBUG [HFileArchiver-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/cdb3383a4019da5f5ef0568dfacd542e/A/8a68e80e8ed744a99ca3865b28ddfe00 to hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/archive/data/default/TestAcidGuarantees/cdb3383a4019da5f5ef0568dfacd542e/A/8a68e80e8ed744a99ca3865b28ddfe00 2024-11-24T22:15:21,464 DEBUG [HFileArchiver-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/cdb3383a4019da5f5ef0568dfacd542e/A/cb02947752e140208717d48bcffc96e9 to hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/archive/data/default/TestAcidGuarantees/cdb3383a4019da5f5ef0568dfacd542e/A/cb02947752e140208717d48bcffc96e9 2024-11-24T22:15:21,466 DEBUG [HFileArchiver-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/cdb3383a4019da5f5ef0568dfacd542e/B/4aeb173214884c898e278f7511d57ee5 to hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/archive/data/default/TestAcidGuarantees/cdb3383a4019da5f5ef0568dfacd542e/B/4aeb173214884c898e278f7511d57ee5 2024-11-24T22:15:21,467 DEBUG [HFileArchiver-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/cdb3383a4019da5f5ef0568dfacd542e/B/4dda70a002ac4948bc84c4e9e55279ee to hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/archive/data/default/TestAcidGuarantees/cdb3383a4019da5f5ef0568dfacd542e/B/4dda70a002ac4948bc84c4e9e55279ee 2024-11-24T22:15:21,468 DEBUG [HFileArchiver-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/cdb3383a4019da5f5ef0568dfacd542e/B/a2bb898be65d4dd3b1b02687a0ba9245 to hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/archive/data/default/TestAcidGuarantees/cdb3383a4019da5f5ef0568dfacd542e/B/a2bb898be65d4dd3b1b02687a0ba9245 2024-11-24T22:15:21,471 DEBUG [HFileArchiver-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/cdb3383a4019da5f5ef0568dfacd542e/C/3fa8093123c44267bf96cab6f4dbd200 to hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/archive/data/default/TestAcidGuarantees/cdb3383a4019da5f5ef0568dfacd542e/C/3fa8093123c44267bf96cab6f4dbd200 2024-11-24T22:15:21,472 DEBUG [HFileArchiver-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/cdb3383a4019da5f5ef0568dfacd542e/C/74528b20d68f43b897f2d61fe8f9f500 to hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/archive/data/default/TestAcidGuarantees/cdb3383a4019da5f5ef0568dfacd542e/C/74528b20d68f43b897f2d61fe8f9f500 2024-11-24T22:15:21,474 DEBUG [HFileArchiver-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/cdb3383a4019da5f5ef0568dfacd542e/C/9f1fc6c1560e45578c0ce54cee5c4bca to hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/archive/data/default/TestAcidGuarantees/cdb3383a4019da5f5ef0568dfacd542e/C/9f1fc6c1560e45578c0ce54cee5c4bca 2024-11-24T22:15:21,478 DEBUG [HFileArchiver-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/cdb3383a4019da5f5ef0568dfacd542e/recovered.edits/498.seqid to hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/archive/data/default/TestAcidGuarantees/cdb3383a4019da5f5ef0568dfacd542e/recovered.edits/498.seqid 2024-11-24T22:15:21,479 DEBUG [HFileArchiver-3 {}] backup.HFileArchiver(610): Deleted hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/cdb3383a4019da5f5ef0568dfacd542e 2024-11-24T22:15:21,479 DEBUG [PEWorker-3 {}] procedure.DeleteTableProcedure(313): Archived TestAcidGuarantees regions 2024-11-24T22:15:21,489 DEBUG [PEWorker-3 {}] procedure.DeleteTableProcedure(120): Deleting regions from META for pid=88, state=RUNNABLE:DELETE_TABLE_REMOVE_FROM_META, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-24T22:15:21,494 WARN [PEWorker-3 {}] procedure.DeleteTableProcedure(370): Deleting some vestigial 1 rows of TestAcidGuarantees from hbase:meta 2024-11-24T22:15:21,497 DEBUG [PEWorker-3 {}] procedure.DeleteTableProcedure(407): Removing 'TestAcidGuarantees' descriptor. 2024-11-24T22:15:21,498 DEBUG [PEWorker-3 {}] procedure.DeleteTableProcedure(126): Deleting assignment state for pid=88, state=RUNNABLE:DELETE_TABLE_UNASSIGN_REGIONS, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-24T22:15:21,498 DEBUG [PEWorker-3 {}] procedure.DeleteTableProcedure(397): Removing 'TestAcidGuarantees' from region states. 2024-11-24T22:15:21,498 DEBUG [PEWorker-3 {}] assignment.RegionStateStore(723): Delete {"totalColumns":1,"row":"TestAcidGuarantees,,1732486488674.cdb3383a4019da5f5ef0568dfacd542e.","families":{"info":[{"qualifier":"","vlen":0,"tag":[],"timestamp":"1732486521498"}]},"ts":"9223372036854775807"} 2024-11-24T22:15:21,500 INFO [PEWorker-3 {}] assignment.RegionStateStore(562): Deleted 1 regions from META 2024-11-24T22:15:21,500 DEBUG [PEWorker-3 {}] assignment.RegionStateStore(563): Deleted regions: [{ENCODED => cdb3383a4019da5f5ef0568dfacd542e, NAME => 'TestAcidGuarantees,,1732486488674.cdb3383a4019da5f5ef0568dfacd542e.', STARTKEY => '', ENDKEY => ''}] 2024-11-24T22:15:21,500 DEBUG [PEWorker-3 {}] procedure.DeleteTableProcedure(401): Marking 'TestAcidGuarantees' as deleted. 2024-11-24T22:15:21,501 DEBUG [PEWorker-3 {}] hbase.MetaTableAccessor(964): Delete {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":0,"tag":[],"timestamp":"1732486521500"}]},"ts":"9223372036854775807"} 2024-11-24T22:15:21,502 INFO [PEWorker-3 {}] hbase.MetaTableAccessor(867): Deleted table TestAcidGuarantees state from META 2024-11-24T22:15:21,503 DEBUG [PEWorker-3 {}] procedure.DeleteTableProcedure(133): Finished pid=88, state=RUNNABLE:DELETE_TABLE_POST_OPERATION, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-24T22:15:21,505 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=88, state=SUCCESS, hasLock=false; DeleteTableProcedure table=TestAcidGuarantees in 51 msec 2024-11-24T22:15:21,561 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37701 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=88 2024-11-24T22:15:21,561 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocator(219): Clear meta cache for TestAcidGuarantees 2024-11-24T22:15:21,562 INFO [RPCClient-NioEventLoopGroup-4-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: DELETE, Table Name: default:TestAcidGuarantees completed 2024-11-24T22:15:21,597 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: TestAcidGuaranteesWithEagerPolicy#testGetAtomicity Thread=248 (was 253), OpenFileDescriptor=453 (was 465), MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=374 (was 472), ProcessCount=11 (was 11), AvailableMemoryMB=5960 (was 6704) 2024-11-24T22:15:21,609 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: TestAcidGuaranteesWithEagerPolicy#testMobScanAtomicity Thread=248, OpenFileDescriptor=455, MaxFileDescriptor=1048576, SystemLoadAverage=374, ProcessCount=11, AvailableMemoryMB=5959 2024-11-24T22:15:21,610 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37701 {}] util.TableDescriptorChecker(321): MEMSTORE_FLUSHSIZE for table descriptor or "hbase.hregion.memstore.flush.size" (131072) is too small, which might cause very frequent flushing. 2024-11-24T22:15:21,611 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37701 {}] master.HMaster$4(2454): Client=jenkins//172.17.0.2 create 'TestAcidGuarantees', {TABLE_ATTRIBUTES => {METADATA => {'hbase.hregion.compacting.memstore.type' => 'EAGER', 'hbase.hregion.memstore.mslab.enabled' => 'false', 'hbase.memstore.inmemoryflush.threshold.factor' => '0.9'}}}, {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-11-24T22:15:21,612 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37701 {}] procedure2.ProcedureExecutor(1139): Stored pid=89, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=false; CreateTableProcedure table=TestAcidGuarantees 2024-11-24T22:15:21,613 INFO [PEWorker-1 {}] procedure.CreateTableProcedure(89): pid=89, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_PRE_OPERATION 2024-11-24T22:15:21,614 DEBUG [PEWorker-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T22:15:21,614 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37701 {}] master.MasterRpcServices(787): Client=jenkins//172.17.0.2 procedure request for creating table: namespace: "default" qualifier: "TestAcidGuarantees" procId is: 89 2024-11-24T22:15:21,614 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37701 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=89 2024-11-24T22:15:21,615 INFO [PEWorker-1 {}] procedure.CreateTableProcedure(89): pid=89, state=RUNNABLE:CREATE_TABLE_WRITE_FS_LAYOUT, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_WRITE_FS_LAYOUT 2024-11-24T22:15:21,620 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45929 is added to blk_1073742174_1350 (size=1061) 2024-11-24T22:15:21,721 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37701 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=89 2024-11-24T22:15:21,930 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37701 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=89 2024-11-24T22:15:22,022 INFO [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(7572): creating {ENCODED => 94f3ccf9f8a638fa974e1f69627832ef, NAME => 'TestAcidGuarantees,,1732486521610.94f3ccf9f8a638fa974e1f69627832ef.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='TestAcidGuarantees', {TABLE_ATTRIBUTES => {METADATA => {'hbase.hregion.compacting.memstore.type' => 'EAGER', 'hbase.hregion.memstore.mslab.enabled' => 'false', 'hbase.memstore.inmemoryflush.threshold.factor' => '0.9', 'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc 2024-11-24T22:15:22,026 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45929 is added to blk_1073742175_1351 (size=53) 2024-11-24T22:15:22,241 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37701 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=89 2024-11-24T22:15:22,427 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(898): Instantiated TestAcidGuarantees,,1732486521610.94f3ccf9f8a638fa974e1f69627832ef.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-24T22:15:22,427 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1722): Closing 94f3ccf9f8a638fa974e1f69627832ef, disabling compactions & flushes 2024-11-24T22:15:22,427 INFO [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1755): Closing region TestAcidGuarantees,,1732486521610.94f3ccf9f8a638fa974e1f69627832ef. 2024-11-24T22:15:22,427 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1776): Time limited wait for close lock on TestAcidGuarantees,,1732486521610.94f3ccf9f8a638fa974e1f69627832ef. 2024-11-24T22:15:22,427 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1843): Acquired close lock on TestAcidGuarantees,,1732486521610.94f3ccf9f8a638fa974e1f69627832ef. after waiting 0 ms 2024-11-24T22:15:22,427 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1853): Updates disabled for region TestAcidGuarantees,,1732486521610.94f3ccf9f8a638fa974e1f69627832ef. 2024-11-24T22:15:22,427 INFO [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1973): Closed TestAcidGuarantees,,1732486521610.94f3ccf9f8a638fa974e1f69627832ef. 2024-11-24T22:15:22,428 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1676): Region close journal for 94f3ccf9f8a638fa974e1f69627832ef: Waiting for close lock at 1732486522427Disabling compacts and flushes for region at 1732486522427Disabling writes for close at 1732486522427Writing region close event to WAL at 1732486522427Closed at 1732486522427 2024-11-24T22:15:22,429 INFO [PEWorker-1 {}] procedure.CreateTableProcedure(89): pid=89, state=RUNNABLE:CREATE_TABLE_ADD_TO_META, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_ADD_TO_META 2024-11-24T22:15:22,429 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":2,"row":"TestAcidGuarantees,,1732486521610.94f3ccf9f8a638fa974e1f69627832ef.","families":{"info":[{"qualifier":"regioninfo","vlen":52,"tag":[],"timestamp":"1732486522429"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1732486522429"}]},"ts":"1732486522429"} 2024-11-24T22:15:22,430 INFO [PEWorker-1 {}] hbase.MetaTableAccessor(832): Added 1 regions to meta. 2024-11-24T22:15:22,431 INFO [PEWorker-1 {}] procedure.CreateTableProcedure(89): pid=89, state=RUNNABLE:CREATE_TABLE_ASSIGN_REGIONS, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_ASSIGN_REGIONS 2024-11-24T22:15:22,431 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1732486522431"}]},"ts":"1732486522431"} 2024-11-24T22:15:22,432 INFO [PEWorker-1 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=ENABLING in hbase:meta 2024-11-24T22:15:22,432 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=90, ppid=89, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=94f3ccf9f8a638fa974e1f69627832ef, ASSIGN}] 2024-11-24T22:15:22,433 INFO [PEWorker-2 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=90, ppid=89, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=94f3ccf9f8a638fa974e1f69627832ef, ASSIGN 2024-11-24T22:15:22,434 INFO [PEWorker-2 {}] assignment.TransitRegionStateProcedure(269): Starting pid=90, ppid=89, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=94f3ccf9f8a638fa974e1f69627832ef, ASSIGN; state=OFFLINE, location=bd6592c37bb2,40087,1732486423713; forceNewPlan=false, retain=false 2024-11-24T22:15:22,584 INFO [PEWorker-4 {}] assignment.RegionStateStore(223): pid=90 updating hbase:meta row=94f3ccf9f8a638fa974e1f69627832ef, regionState=OPENING, regionLocation=bd6592c37bb2,40087,1732486423713 2024-11-24T22:15:22,586 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=90, ppid=89, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=94f3ccf9f8a638fa974e1f69627832ef, ASSIGN because future has completed 2024-11-24T22:15:22,586 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=91, ppid=90, state=RUNNABLE, hasLock=false; OpenRegionProcedure 94f3ccf9f8a638fa974e1f69627832ef, server=bd6592c37bb2,40087,1732486423713}] 2024-11-24T22:15:22,741 INFO [RS_OPEN_REGION-regionserver/bd6592c37bb2:0-0 {event_type=M_RS_OPEN_REGION, pid=91}] handler.AssignRegionHandler(132): Open TestAcidGuarantees,,1732486521610.94f3ccf9f8a638fa974e1f69627832ef. 2024-11-24T22:15:22,741 DEBUG [RS_OPEN_REGION-regionserver/bd6592c37bb2:0-0 {event_type=M_RS_OPEN_REGION, pid=91}] regionserver.HRegion(7752): Opening region: {ENCODED => 94f3ccf9f8a638fa974e1f69627832ef, NAME => 'TestAcidGuarantees,,1732486521610.94f3ccf9f8a638fa974e1f69627832ef.', STARTKEY => '', ENDKEY => ''} 2024-11-24T22:15:22,741 DEBUG [RS_OPEN_REGION-regionserver/bd6592c37bb2:0-0 {event_type=M_RS_OPEN_REGION, pid=91}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table TestAcidGuarantees 94f3ccf9f8a638fa974e1f69627832ef 2024-11-24T22:15:22,741 DEBUG [RS_OPEN_REGION-regionserver/bd6592c37bb2:0-0 {event_type=M_RS_OPEN_REGION, pid=91}] regionserver.HRegion(898): Instantiated TestAcidGuarantees,,1732486521610.94f3ccf9f8a638fa974e1f69627832ef.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-24T22:15:22,742 DEBUG [RS_OPEN_REGION-regionserver/bd6592c37bb2:0-0 {event_type=M_RS_OPEN_REGION, pid=91}] regionserver.HRegion(7794): checking encryption for 94f3ccf9f8a638fa974e1f69627832ef 2024-11-24T22:15:22,742 DEBUG [RS_OPEN_REGION-regionserver/bd6592c37bb2:0-0 {event_type=M_RS_OPEN_REGION, pid=91}] regionserver.HRegion(7797): checking classloading for 94f3ccf9f8a638fa974e1f69627832ef 2024-11-24T22:15:22,743 INFO [StoreOpener-94f3ccf9f8a638fa974e1f69627832ef-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family A of region 94f3ccf9f8a638fa974e1f69627832ef 2024-11-24T22:15:22,744 INFO [StoreOpener-94f3ccf9f8a638fa974e1f69627832ef-1 {}] regionserver.CompactingMemStore(122): Store=A, in-memory flush size threshold=38.40 KB, immutable segments index type=ARRAY_MAP, compactor=EAGER, pipelineThreshold=2, compactionCellMax=10 2024-11-24T22:15:22,744 INFO [StoreOpener-94f3ccf9f8a638fa974e1f69627832ef-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 94f3ccf9f8a638fa974e1f69627832ef columnFamilyName A 2024-11-24T22:15:22,744 DEBUG [StoreOpener-94f3ccf9f8a638fa974e1f69627832ef-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T22:15:22,744 INFO [StoreOpener-94f3ccf9f8a638fa974e1f69627832ef-1 {}] regionserver.HStore(327): Store=94f3ccf9f8a638fa974e1f69627832ef/A, memstore type=CompactingMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-24T22:15:22,744 INFO [StoreOpener-94f3ccf9f8a638fa974e1f69627832ef-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family B of region 94f3ccf9f8a638fa974e1f69627832ef 2024-11-24T22:15:22,745 INFO [StoreOpener-94f3ccf9f8a638fa974e1f69627832ef-1 {}] regionserver.CompactingMemStore(122): Store=B, in-memory flush size threshold=38.40 KB, immutable segments index type=ARRAY_MAP, compactor=EAGER, pipelineThreshold=2, compactionCellMax=10 2024-11-24T22:15:22,745 INFO [StoreOpener-94f3ccf9f8a638fa974e1f69627832ef-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 94f3ccf9f8a638fa974e1f69627832ef columnFamilyName B 2024-11-24T22:15:22,745 DEBUG [StoreOpener-94f3ccf9f8a638fa974e1f69627832ef-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T22:15:22,746 INFO [StoreOpener-94f3ccf9f8a638fa974e1f69627832ef-1 {}] regionserver.HStore(327): Store=94f3ccf9f8a638fa974e1f69627832ef/B, memstore type=CompactingMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-24T22:15:22,746 INFO [StoreOpener-94f3ccf9f8a638fa974e1f69627832ef-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family C of region 94f3ccf9f8a638fa974e1f69627832ef 2024-11-24T22:15:22,747 INFO [StoreOpener-94f3ccf9f8a638fa974e1f69627832ef-1 {}] regionserver.CompactingMemStore(122): Store=C, in-memory flush size threshold=38.40 KB, immutable segments index type=ARRAY_MAP, compactor=EAGER, pipelineThreshold=2, compactionCellMax=10 2024-11-24T22:15:22,747 INFO [StoreOpener-94f3ccf9f8a638fa974e1f69627832ef-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 94f3ccf9f8a638fa974e1f69627832ef columnFamilyName C 2024-11-24T22:15:22,747 DEBUG [StoreOpener-94f3ccf9f8a638fa974e1f69627832ef-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T22:15:22,747 INFO [StoreOpener-94f3ccf9f8a638fa974e1f69627832ef-1 {}] regionserver.HStore(327): Store=94f3ccf9f8a638fa974e1f69627832ef/C, memstore type=CompactingMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-24T22:15:22,747 INFO [RS_OPEN_REGION-regionserver/bd6592c37bb2:0-0 {event_type=M_RS_OPEN_REGION, pid=91}] regionserver.HRegion(1212): Setting FlushNonSloppyStoresFirstPolicy for the region=TestAcidGuarantees,,1732486521610.94f3ccf9f8a638fa974e1f69627832ef. 2024-11-24T22:15:22,747 DEBUG [RS_OPEN_REGION-regionserver/bd6592c37bb2:0-0 {event_type=M_RS_OPEN_REGION, pid=91}] regionserver.HRegion(1038): replaying wal for 94f3ccf9f8a638fa974e1f69627832ef 2024-11-24T22:15:22,748 DEBUG [RS_OPEN_REGION-regionserver/bd6592c37bb2:0-0 {event_type=M_RS_OPEN_REGION, pid=91}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/94f3ccf9f8a638fa974e1f69627832ef 2024-11-24T22:15:22,748 DEBUG [RS_OPEN_REGION-regionserver/bd6592c37bb2:0-0 {event_type=M_RS_OPEN_REGION, pid=91}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/94f3ccf9f8a638fa974e1f69627832ef 2024-11-24T22:15:22,749 DEBUG [RS_OPEN_REGION-regionserver/bd6592c37bb2:0-0 {event_type=M_RS_OPEN_REGION, pid=91}] regionserver.HRegion(1048): stopping wal replay for 94f3ccf9f8a638fa974e1f69627832ef 2024-11-24T22:15:22,749 DEBUG [RS_OPEN_REGION-regionserver/bd6592c37bb2:0-0 {event_type=M_RS_OPEN_REGION, pid=91}] regionserver.HRegion(1060): Cleaning up temporary data for 94f3ccf9f8a638fa974e1f69627832ef 2024-11-24T22:15:22,749 DEBUG [RS_OPEN_REGION-regionserver/bd6592c37bb2:0-0 {event_type=M_RS_OPEN_REGION, pid=91}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table TestAcidGuarantees descriptor;using region.getMemStoreFlushHeapSize/# of families (16.0 M)) instead. 2024-11-24T22:15:22,750 DEBUG [RS_OPEN_REGION-regionserver/bd6592c37bb2:0-0 {event_type=M_RS_OPEN_REGION, pid=91}] regionserver.HRegion(1093): writing seq id for 94f3ccf9f8a638fa974e1f69627832ef 2024-11-24T22:15:22,751 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37701 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=89 2024-11-24T22:15:22,752 DEBUG [RS_OPEN_REGION-regionserver/bd6592c37bb2:0-0 {event_type=M_RS_OPEN_REGION, pid=91}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/94f3ccf9f8a638fa974e1f69627832ef/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-24T22:15:22,752 INFO [RS_OPEN_REGION-regionserver/bd6592c37bb2:0-0 {event_type=M_RS_OPEN_REGION, pid=91}] regionserver.HRegion(1114): Opened 94f3ccf9f8a638fa974e1f69627832ef; next sequenceid=2; ConstantSizeRegionSplitPolicy{desiredMaxFileSize=74437587, jitterRate=0.10920648276805878}, FlushLargeStoresPolicy{flushSizeLowerBound=16777216} 2024-11-24T22:15:22,752 DEBUG [RS_OPEN_REGION-regionserver/bd6592c37bb2:0-0 {event_type=M_RS_OPEN_REGION, pid=91}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 94f3ccf9f8a638fa974e1f69627832ef 2024-11-24T22:15:22,753 DEBUG [RS_OPEN_REGION-regionserver/bd6592c37bb2:0-0 {event_type=M_RS_OPEN_REGION, pid=91}] regionserver.HRegion(1006): Region open journal for 94f3ccf9f8a638fa974e1f69627832ef: Running coprocessor pre-open hook at 1732486522742Writing region info on filesystem at 1732486522742Initializing all the Stores at 1732486522742Instantiating store for column family {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732486522742Instantiating store for column family {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732486522742Instantiating store for column family {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732486522742Cleaning up temporary data from old regions at 1732486522749 (+7 ms)Running coprocessor post-open hooks at 1732486522752 (+3 ms)Region opened successfully at 1732486522753 (+1 ms) 2024-11-24T22:15:22,754 INFO [RS_OPEN_REGION-regionserver/bd6592c37bb2:0-0 {event_type=M_RS_OPEN_REGION, pid=91}] regionserver.HRegionServer(2236): Post open deploy tasks for TestAcidGuarantees,,1732486521610.94f3ccf9f8a638fa974e1f69627832ef., pid=91, masterSystemTime=1732486522737 2024-11-24T22:15:22,755 DEBUG [RS_OPEN_REGION-regionserver/bd6592c37bb2:0-0 {event_type=M_RS_OPEN_REGION, pid=91}] regionserver.HRegionServer(2266): Finished post open deploy task for TestAcidGuarantees,,1732486521610.94f3ccf9f8a638fa974e1f69627832ef. 2024-11-24T22:15:22,755 INFO [RS_OPEN_REGION-regionserver/bd6592c37bb2:0-0 {event_type=M_RS_OPEN_REGION, pid=91}] handler.AssignRegionHandler(153): Opened TestAcidGuarantees,,1732486521610.94f3ccf9f8a638fa974e1f69627832ef. 2024-11-24T22:15:22,756 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=90 updating hbase:meta row=94f3ccf9f8a638fa974e1f69627832ef, regionState=OPEN, openSeqNum=2, regionLocation=bd6592c37bb2,40087,1732486423713 2024-11-24T22:15:22,757 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=91, ppid=90, state=RUNNABLE, hasLock=false; OpenRegionProcedure 94f3ccf9f8a638fa974e1f69627832ef, server=bd6592c37bb2,40087,1732486423713 because future has completed 2024-11-24T22:15:22,759 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=91, resume processing ppid=90 2024-11-24T22:15:22,759 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=91, ppid=90, state=SUCCESS, hasLock=false; OpenRegionProcedure 94f3ccf9f8a638fa974e1f69627832ef, server=bd6592c37bb2,40087,1732486423713 in 172 msec 2024-11-24T22:15:22,761 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=90, resume processing ppid=89 2024-11-24T22:15:22,761 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=90, ppid=89, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=94f3ccf9f8a638fa974e1f69627832ef, ASSIGN in 327 msec 2024-11-24T22:15:22,762 INFO [PEWorker-5 {}] procedure.CreateTableProcedure(89): pid=89, state=RUNNABLE:CREATE_TABLE_UPDATE_DESC_CACHE, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_UPDATE_DESC_CACHE 2024-11-24T22:15:22,762 DEBUG [PEWorker-5 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1732486522762"}]},"ts":"1732486522762"} 2024-11-24T22:15:22,763 INFO [PEWorker-5 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=ENABLED in hbase:meta 2024-11-24T22:15:22,764 INFO [PEWorker-5 {}] procedure.CreateTableProcedure(89): pid=89, state=RUNNABLE:CREATE_TABLE_POST_OPERATION, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_POST_OPERATION 2024-11-24T22:15:22,765 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=89, state=SUCCESS, hasLock=false; CreateTableProcedure table=TestAcidGuarantees in 1.1530 sec 2024-11-24T22:15:23,761 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37701 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=89 2024-11-24T22:15:23,761 INFO [RPCClient-NioEventLoopGroup-4-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: CREATE, Table Name: default:TestAcidGuarantees completed 2024-11-24T22:15:23,762 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@7befe57f, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-24T22:15:23,762 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request bd6592c37bb2,37701,-1 for getting cluster id 2024-11-24T22:15:23,762 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-24T22:15:23,763 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '3c461ac1-06fb-4739-8702-6368e685b03c' 2024-11-24T22:15:23,763 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-24T22:15:23,763 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "3c461ac1-06fb-4739-8702-6368e685b03c" 2024-11-24T22:15:23,763 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@72c71396, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-24T22:15:23,764 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [bd6592c37bb2,37701,-1] 2024-11-24T22:15:23,764 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-24T22:15:23,764 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-24T22:15:23,765 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:33888, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-24T22:15:23,765 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@7b86eecf, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-24T22:15:23,765 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-24T22:15:23,766 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=bd6592c37bb2,40087,1732486423713, seqNum=-1] 2024-11-24T22:15:23,766 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-24T22:15:23,767 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:60998, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-24T22:15:23,768 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching master stub from registry 2024-11-24T22:15:23,768 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncConnectionImpl(321): The fetched master address is bd6592c37bb2,37701,1732486422992 2024-11-24T22:15:23,768 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.ConnectionUtils(555): The fetched master stub is org.apache.hadoop.hbase.shaded.protobuf.generated.MasterProtos$MasterService$Stub@17a420d5 2024-11-24T22:15:23,768 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=MasterService, sasl=false 2024-11-24T22:15:23,769 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:33896, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=MasterService 2024-11-24T22:15:23,770 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37701 {}] util.TableDescriptorChecker(321): MEMSTORE_FLUSHSIZE for table descriptor or "hbase.hregion.memstore.flush.size" (131072) is too small, which might cause very frequent flushing. 2024-11-24T22:15:23,771 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37701 {}] master.HMaster$14(2863): Client=jenkins//172.17.0.2 modify table TestAcidGuarantees from 'TestAcidGuarantees', {TABLE_ATTRIBUTES => {METADATA => {'hbase.hregion.compacting.memstore.type' => 'EAGER', 'hbase.hregion.memstore.mslab.enabled' => 'false', 'hbase.memstore.inmemoryflush.threshold.factor' => '0.9', 'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} to 'TestAcidGuarantees', {TABLE_ATTRIBUTES => {METADATA => {'hbase.hregion.compacting.memstore.type' => 'EAGER', 'hbase.hregion.memstore.mslab.enabled' => 'false', 'hbase.memstore.inmemoryflush.threshold.factor' => '0.9', 'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', MOB_THRESHOLD => '4', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', IS_MOB => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-11-24T22:15:23,772 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37701 {}] procedure2.ProcedureExecutor(1139): Stored pid=92, state=RUNNABLE:MODIFY_TABLE_PREPARE, hasLock=false; ModifyTableProcedure table=TestAcidGuarantees 2024-11-24T22:15:23,780 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45929 is added to blk_1073742176_1352 (size=1097) 2024-11-24T22:15:24,181 DEBUG [PEWorker-3 {}] util.FSTableDescriptors(591): Deleted hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/.tabledesc/.tableinfo.0000000001.1061 2024-11-24T22:15:24,181 INFO [PEWorker-3 {}] util.FSTableDescriptors(361): Updated tableinfo=hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/.tabledesc/.tableinfo.0000000002.1097 2024-11-24T22:15:24,183 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=93, ppid=92, state=RUNNABLE:REOPEN_TABLE_REGIONS_GET_REGIONS, hasLock=false; ReopenTableRegionsProcedure table=TestAcidGuarantees}] 2024-11-24T22:15:24,185 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=94, ppid=93, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=94f3ccf9f8a638fa974e1f69627832ef, REOPEN/MOVE}] 2024-11-24T22:15:24,186 INFO [PEWorker-2 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=94, ppid=93, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=94f3ccf9f8a638fa974e1f69627832ef, REOPEN/MOVE 2024-11-24T22:15:24,186 INFO [PEWorker-2 {}] assignment.RegionStateStore(223): pid=94 updating hbase:meta row=94f3ccf9f8a638fa974e1f69627832ef, regionState=CLOSING, regionLocation=bd6592c37bb2,40087,1732486423713 2024-11-24T22:15:24,187 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=94, ppid=93, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=94f3ccf9f8a638fa974e1f69627832ef, REOPEN/MOVE because future has completed 2024-11-24T22:15:24,188 DEBUG [PEWorker-4 {}] assignment.TransitRegionStateProcedure(375): Close region: isSplit: false: evictOnSplit: true: evictOnClose: false 2024-11-24T22:15:24,188 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=95, ppid=94, state=RUNNABLE, hasLock=false; CloseRegionProcedure 94f3ccf9f8a638fa974e1f69627832ef, server=bd6592c37bb2,40087,1732486423713}] 2024-11-24T22:15:24,340 INFO [RS_CLOSE_REGION-regionserver/bd6592c37bb2:0-0 {event_type=M_RS_CLOSE_REGION, pid=95}] handler.UnassignRegionHandler(122): Close 94f3ccf9f8a638fa974e1f69627832ef 2024-11-24T22:15:24,340 DEBUG [RS_CLOSE_REGION-regionserver/bd6592c37bb2:0-0 {event_type=M_RS_CLOSE_REGION, pid=95}] handler.UnassignRegionHandler(136): Unassign region: split region: false: evictCache: false 2024-11-24T22:15:24,340 DEBUG [RS_CLOSE_REGION-regionserver/bd6592c37bb2:0-0 {event_type=M_RS_CLOSE_REGION, pid=95}] regionserver.HRegion(1722): Closing 94f3ccf9f8a638fa974e1f69627832ef, disabling compactions & flushes 2024-11-24T22:15:24,340 INFO [RS_CLOSE_REGION-regionserver/bd6592c37bb2:0-0 {event_type=M_RS_CLOSE_REGION, pid=95}] regionserver.HRegion(1755): Closing region TestAcidGuarantees,,1732486521610.94f3ccf9f8a638fa974e1f69627832ef. 2024-11-24T22:15:24,340 DEBUG [RS_CLOSE_REGION-regionserver/bd6592c37bb2:0-0 {event_type=M_RS_CLOSE_REGION, pid=95}] regionserver.HRegion(1776): Time limited wait for close lock on TestAcidGuarantees,,1732486521610.94f3ccf9f8a638fa974e1f69627832ef. 2024-11-24T22:15:24,340 DEBUG [RS_CLOSE_REGION-regionserver/bd6592c37bb2:0-0 {event_type=M_RS_CLOSE_REGION, pid=95}] regionserver.HRegion(1843): Acquired close lock on TestAcidGuarantees,,1732486521610.94f3ccf9f8a638fa974e1f69627832ef. after waiting 0 ms 2024-11-24T22:15:24,340 DEBUG [RS_CLOSE_REGION-regionserver/bd6592c37bb2:0-0 {event_type=M_RS_CLOSE_REGION, pid=95}] regionserver.HRegion(1853): Updates disabled for region TestAcidGuarantees,,1732486521610.94f3ccf9f8a638fa974e1f69627832ef. 2024-11-24T22:15:24,343 DEBUG [RS_CLOSE_REGION-regionserver/bd6592c37bb2:0-0 {event_type=M_RS_CLOSE_REGION, pid=95}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/94f3ccf9f8a638fa974e1f69627832ef/recovered.edits/4.seqid, newMaxSeqId=4, maxSeqId=1 2024-11-24T22:15:24,343 INFO [RS_CLOSE_REGION-regionserver/bd6592c37bb2:0-0 {event_type=M_RS_CLOSE_REGION, pid=95}] regionserver.HRegion(1973): Closed TestAcidGuarantees,,1732486521610.94f3ccf9f8a638fa974e1f69627832ef. 2024-11-24T22:15:24,344 DEBUG [RS_CLOSE_REGION-regionserver/bd6592c37bb2:0-0 {event_type=M_RS_CLOSE_REGION, pid=95}] regionserver.HRegion(1676): Region close journal for 94f3ccf9f8a638fa974e1f69627832ef: Waiting for close lock at 1732486524340Running coprocessor pre-close hooks at 1732486524340Disabling compacts and flushes for region at 1732486524340Disabling writes for close at 1732486524340Writing region close event to WAL at 1732486524340Running coprocessor post-close hooks at 1732486524343 (+3 ms)Closed at 1732486524343 2024-11-24T22:15:24,344 WARN [RS_CLOSE_REGION-regionserver/bd6592c37bb2:0-0 {event_type=M_RS_CLOSE_REGION, pid=95}] regionserver.HRegionServer(3299): Not adding moved region record: 94f3ccf9f8a638fa974e1f69627832ef to self. 2024-11-24T22:15:24,345 INFO [RS_CLOSE_REGION-regionserver/bd6592c37bb2:0-0 {event_type=M_RS_CLOSE_REGION, pid=95}] handler.UnassignRegionHandler(157): Closed 94f3ccf9f8a638fa974e1f69627832ef 2024-11-24T22:15:24,346 INFO [PEWorker-3 {}] assignment.RegionStateStore(223): pid=94 updating hbase:meta row=94f3ccf9f8a638fa974e1f69627832ef, regionState=CLOSED 2024-11-24T22:15:24,347 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=95, ppid=94, state=RUNNABLE, hasLock=false; CloseRegionProcedure 94f3ccf9f8a638fa974e1f69627832ef, server=bd6592c37bb2,40087,1732486423713 because future has completed 2024-11-24T22:15:24,349 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=95, resume processing ppid=94 2024-11-24T22:15:24,349 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=95, ppid=94, state=SUCCESS, hasLock=false; CloseRegionProcedure 94f3ccf9f8a638fa974e1f69627832ef, server=bd6592c37bb2,40087,1732486423713 in 159 msec 2024-11-24T22:15:24,349 INFO [PEWorker-2 {}] assignment.TransitRegionStateProcedure(269): Starting pid=94, ppid=93, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=94f3ccf9f8a638fa974e1f69627832ef, REOPEN/MOVE; state=CLOSED, location=bd6592c37bb2,40087,1732486423713; forceNewPlan=false, retain=true 2024-11-24T22:15:24,500 INFO [PEWorker-5 {}] assignment.RegionStateStore(223): pid=94 updating hbase:meta row=94f3ccf9f8a638fa974e1f69627832ef, regionState=OPENING, regionLocation=bd6592c37bb2,40087,1732486423713 2024-11-24T22:15:24,501 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=94, ppid=93, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=94f3ccf9f8a638fa974e1f69627832ef, REOPEN/MOVE because future has completed 2024-11-24T22:15:24,501 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=96, ppid=94, state=RUNNABLE, hasLock=false; OpenRegionProcedure 94f3ccf9f8a638fa974e1f69627832ef, server=bd6592c37bb2,40087,1732486423713}] 2024-11-24T22:15:24,656 INFO [RS_OPEN_REGION-regionserver/bd6592c37bb2:0-0 {event_type=M_RS_OPEN_REGION, pid=96}] handler.AssignRegionHandler(132): Open TestAcidGuarantees,,1732486521610.94f3ccf9f8a638fa974e1f69627832ef. 2024-11-24T22:15:24,656 DEBUG [RS_OPEN_REGION-regionserver/bd6592c37bb2:0-0 {event_type=M_RS_OPEN_REGION, pid=96}] regionserver.HRegion(7752): Opening region: {ENCODED => 94f3ccf9f8a638fa974e1f69627832ef, NAME => 'TestAcidGuarantees,,1732486521610.94f3ccf9f8a638fa974e1f69627832ef.', STARTKEY => '', ENDKEY => ''} 2024-11-24T22:15:24,656 DEBUG [RS_OPEN_REGION-regionserver/bd6592c37bb2:0-0 {event_type=M_RS_OPEN_REGION, pid=96}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table TestAcidGuarantees 94f3ccf9f8a638fa974e1f69627832ef 2024-11-24T22:15:24,656 DEBUG [RS_OPEN_REGION-regionserver/bd6592c37bb2:0-0 {event_type=M_RS_OPEN_REGION, pid=96}] regionserver.HRegion(898): Instantiated TestAcidGuarantees,,1732486521610.94f3ccf9f8a638fa974e1f69627832ef.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-24T22:15:24,656 DEBUG [RS_OPEN_REGION-regionserver/bd6592c37bb2:0-0 {event_type=M_RS_OPEN_REGION, pid=96}] regionserver.HRegion(7794): checking encryption for 94f3ccf9f8a638fa974e1f69627832ef 2024-11-24T22:15:24,656 DEBUG [RS_OPEN_REGION-regionserver/bd6592c37bb2:0-0 {event_type=M_RS_OPEN_REGION, pid=96}] regionserver.HRegion(7797): checking classloading for 94f3ccf9f8a638fa974e1f69627832ef 2024-11-24T22:15:24,657 INFO [StoreOpener-94f3ccf9f8a638fa974e1f69627832ef-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family A of region 94f3ccf9f8a638fa974e1f69627832ef 2024-11-24T22:15:24,658 INFO [StoreOpener-94f3ccf9f8a638fa974e1f69627832ef-1 {}] regionserver.CompactingMemStore(122): Store=A, in-memory flush size threshold=38.40 KB, immutable segments index type=ARRAY_MAP, compactor=EAGER, pipelineThreshold=2, compactionCellMax=10 2024-11-24T22:15:24,658 INFO [StoreOpener-94f3ccf9f8a638fa974e1f69627832ef-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 94f3ccf9f8a638fa974e1f69627832ef columnFamilyName A 2024-11-24T22:15:24,659 DEBUG [StoreOpener-94f3ccf9f8a638fa974e1f69627832ef-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T22:15:24,659 INFO [StoreOpener-94f3ccf9f8a638fa974e1f69627832ef-1 {}] regionserver.HStore(327): Store=94f3ccf9f8a638fa974e1f69627832ef/A, memstore type=CompactingMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-24T22:15:24,660 INFO [StoreOpener-94f3ccf9f8a638fa974e1f69627832ef-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family B of region 94f3ccf9f8a638fa974e1f69627832ef 2024-11-24T22:15:24,660 INFO [StoreOpener-94f3ccf9f8a638fa974e1f69627832ef-1 {}] regionserver.CompactingMemStore(122): Store=B, in-memory flush size threshold=38.40 KB, immutable segments index type=ARRAY_MAP, compactor=EAGER, pipelineThreshold=2, compactionCellMax=10 2024-11-24T22:15:24,660 INFO [StoreOpener-94f3ccf9f8a638fa974e1f69627832ef-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 94f3ccf9f8a638fa974e1f69627832ef columnFamilyName B 2024-11-24T22:15:24,660 DEBUG [StoreOpener-94f3ccf9f8a638fa974e1f69627832ef-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T22:15:24,661 INFO [StoreOpener-94f3ccf9f8a638fa974e1f69627832ef-1 {}] regionserver.HStore(327): Store=94f3ccf9f8a638fa974e1f69627832ef/B, memstore type=CompactingMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-24T22:15:24,661 INFO [StoreOpener-94f3ccf9f8a638fa974e1f69627832ef-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family C of region 94f3ccf9f8a638fa974e1f69627832ef 2024-11-24T22:15:24,661 INFO [StoreOpener-94f3ccf9f8a638fa974e1f69627832ef-1 {}] regionserver.CompactingMemStore(122): Store=C, in-memory flush size threshold=38.40 KB, immutable segments index type=ARRAY_MAP, compactor=EAGER, pipelineThreshold=2, compactionCellMax=10 2024-11-24T22:15:24,661 INFO [StoreOpener-94f3ccf9f8a638fa974e1f69627832ef-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 94f3ccf9f8a638fa974e1f69627832ef columnFamilyName C 2024-11-24T22:15:24,661 DEBUG [StoreOpener-94f3ccf9f8a638fa974e1f69627832ef-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T22:15:24,662 INFO [StoreOpener-94f3ccf9f8a638fa974e1f69627832ef-1 {}] regionserver.HStore(327): Store=94f3ccf9f8a638fa974e1f69627832ef/C, memstore type=CompactingMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-24T22:15:24,662 INFO [RS_OPEN_REGION-regionserver/bd6592c37bb2:0-0 {event_type=M_RS_OPEN_REGION, pid=96}] regionserver.HRegion(1212): Setting FlushNonSloppyStoresFirstPolicy for the region=TestAcidGuarantees,,1732486521610.94f3ccf9f8a638fa974e1f69627832ef. 2024-11-24T22:15:24,662 DEBUG [RS_OPEN_REGION-regionserver/bd6592c37bb2:0-0 {event_type=M_RS_OPEN_REGION, pid=96}] regionserver.HRegion(1038): replaying wal for 94f3ccf9f8a638fa974e1f69627832ef 2024-11-24T22:15:24,662 DEBUG [RS_OPEN_REGION-regionserver/bd6592c37bb2:0-0 {event_type=M_RS_OPEN_REGION, pid=96}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/94f3ccf9f8a638fa974e1f69627832ef 2024-11-24T22:15:24,663 DEBUG [RS_OPEN_REGION-regionserver/bd6592c37bb2:0-0 {event_type=M_RS_OPEN_REGION, pid=96}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/94f3ccf9f8a638fa974e1f69627832ef 2024-11-24T22:15:24,664 DEBUG [RS_OPEN_REGION-regionserver/bd6592c37bb2:0-0 {event_type=M_RS_OPEN_REGION, pid=96}] regionserver.HRegion(1048): stopping wal replay for 94f3ccf9f8a638fa974e1f69627832ef 2024-11-24T22:15:24,664 DEBUG [RS_OPEN_REGION-regionserver/bd6592c37bb2:0-0 {event_type=M_RS_OPEN_REGION, pid=96}] regionserver.HRegion(1060): Cleaning up temporary data for 94f3ccf9f8a638fa974e1f69627832ef 2024-11-24T22:15:24,664 DEBUG [RS_OPEN_REGION-regionserver/bd6592c37bb2:0-0 {event_type=M_RS_OPEN_REGION, pid=96}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table TestAcidGuarantees descriptor;using region.getMemStoreFlushHeapSize/# of families (16.0 M)) instead. 2024-11-24T22:15:24,665 DEBUG [RS_OPEN_REGION-regionserver/bd6592c37bb2:0-0 {event_type=M_RS_OPEN_REGION, pid=96}] regionserver.HRegion(1093): writing seq id for 94f3ccf9f8a638fa974e1f69627832ef 2024-11-24T22:15:24,665 INFO [RS_OPEN_REGION-regionserver/bd6592c37bb2:0-0 {event_type=M_RS_OPEN_REGION, pid=96}] regionserver.HRegion(1114): Opened 94f3ccf9f8a638fa974e1f69627832ef; next sequenceid=5; ConstantSizeRegionSplitPolicy{desiredMaxFileSize=61765305, jitterRate=-0.07962523400783539}, FlushLargeStoresPolicy{flushSizeLowerBound=16777216} 2024-11-24T22:15:24,665 DEBUG [RS_OPEN_REGION-regionserver/bd6592c37bb2:0-0 {event_type=M_RS_OPEN_REGION, pid=96}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 94f3ccf9f8a638fa974e1f69627832ef 2024-11-24T22:15:24,666 DEBUG [RS_OPEN_REGION-regionserver/bd6592c37bb2:0-0 {event_type=M_RS_OPEN_REGION, pid=96}] regionserver.HRegion(1006): Region open journal for 94f3ccf9f8a638fa974e1f69627832ef: Running coprocessor pre-open hook at 1732486524656Writing region info on filesystem at 1732486524656Initializing all the Stores at 1732486524657 (+1 ms)Instantiating store for column family {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', MOB_THRESHOLD => '4', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', IS_MOB => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732486524657Instantiating store for column family {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732486524657Instantiating store for column family {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732486524657Cleaning up temporary data from old regions at 1732486524664 (+7 ms)Running coprocessor post-open hooks at 1732486524665 (+1 ms)Region opened successfully at 1732486524666 (+1 ms) 2024-11-24T22:15:24,667 INFO [RS_OPEN_REGION-regionserver/bd6592c37bb2:0-0 {event_type=M_RS_OPEN_REGION, pid=96}] regionserver.HRegionServer(2236): Post open deploy tasks for TestAcidGuarantees,,1732486521610.94f3ccf9f8a638fa974e1f69627832ef., pid=96, masterSystemTime=1732486524653 2024-11-24T22:15:24,668 DEBUG [RS_OPEN_REGION-regionserver/bd6592c37bb2:0-0 {event_type=M_RS_OPEN_REGION, pid=96}] regionserver.HRegionServer(2266): Finished post open deploy task for TestAcidGuarantees,,1732486521610.94f3ccf9f8a638fa974e1f69627832ef. 2024-11-24T22:15:24,668 INFO [RS_OPEN_REGION-regionserver/bd6592c37bb2:0-0 {event_type=M_RS_OPEN_REGION, pid=96}] handler.AssignRegionHandler(153): Opened TestAcidGuarantees,,1732486521610.94f3ccf9f8a638fa974e1f69627832ef. 2024-11-24T22:15:24,668 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=94 updating hbase:meta row=94f3ccf9f8a638fa974e1f69627832ef, regionState=OPEN, openSeqNum=5, regionLocation=bd6592c37bb2,40087,1732486423713 2024-11-24T22:15:24,670 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=96, ppid=94, state=RUNNABLE, hasLock=false; OpenRegionProcedure 94f3ccf9f8a638fa974e1f69627832ef, server=bd6592c37bb2,40087,1732486423713 because future has completed 2024-11-24T22:15:24,671 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=96, resume processing ppid=94 2024-11-24T22:15:24,671 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=96, ppid=94, state=SUCCESS, hasLock=false; OpenRegionProcedure 94f3ccf9f8a638fa974e1f69627832ef, server=bd6592c37bb2,40087,1732486423713 in 169 msec 2024-11-24T22:15:24,673 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=94, resume processing ppid=93 2024-11-24T22:15:24,673 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=94, ppid=93, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=94f3ccf9f8a638fa974e1f69627832ef, REOPEN/MOVE in 486 msec 2024-11-24T22:15:24,675 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=93, resume processing ppid=92 2024-11-24T22:15:24,675 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=93, ppid=92, state=SUCCESS, hasLock=false; ReopenTableRegionsProcedure table=TestAcidGuarantees in 490 msec 2024-11-24T22:15:24,676 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=92, state=SUCCESS, hasLock=false; ModifyTableProcedure table=TestAcidGuarantees in 904 msec 2024-11-24T22:15:24,677 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37701 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=92 2024-11-24T22:15:24,677 INFO [RPCClient-NioEventLoopGroup-4-8 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: MODIFY_COLUMN_FAMILY, Table Name: default:TestAcidGuarantees completed 2024-11-24T22:15:24,678 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@b10b9e8, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-24T22:15:24,678 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request bd6592c37bb2,37701,-1 for getting cluster id 2024-11-24T22:15:24,678 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-24T22:15:24,679 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '3c461ac1-06fb-4739-8702-6368e685b03c' 2024-11-24T22:15:24,679 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-24T22:15:24,679 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "3c461ac1-06fb-4739-8702-6368e685b03c" 2024-11-24T22:15:24,679 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@c9fd4e8, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-24T22:15:24,679 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [bd6592c37bb2,37701,-1] 2024-11-24T22:15:24,679 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-24T22:15:24,679 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-24T22:15:24,680 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:33928, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-24T22:15:24,681 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@596c5e89, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-24T22:15:24,681 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@21c265f0, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-24T22:15:24,681 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request bd6592c37bb2,37701,-1 for getting cluster id 2024-11-24T22:15:24,681 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-24T22:15:24,682 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '3c461ac1-06fb-4739-8702-6368e685b03c' 2024-11-24T22:15:24,682 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-24T22:15:24,682 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "3c461ac1-06fb-4739-8702-6368e685b03c" 2024-11-24T22:15:24,682 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@41c6ea0e, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-24T22:15:24,682 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [bd6592c37bb2,37701,-1] 2024-11-24T22:15:24,682 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-24T22:15:24,683 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-24T22:15:24,683 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:33954, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-24T22:15:24,683 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@5fe5b506, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-24T22:15:24,684 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@c21b87a, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-24T22:15:24,684 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request bd6592c37bb2,37701,-1 for getting cluster id 2024-11-24T22:15:24,684 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-24T22:15:24,685 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '3c461ac1-06fb-4739-8702-6368e685b03c' 2024-11-24T22:15:24,685 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-24T22:15:24,685 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "3c461ac1-06fb-4739-8702-6368e685b03c" 2024-11-24T22:15:24,685 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@746cf492, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-24T22:15:24,685 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [bd6592c37bb2,37701,-1] 2024-11-24T22:15:24,685 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-24T22:15:24,685 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-24T22:15:24,686 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:33968, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-24T22:15:24,686 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@2793ea97, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-24T22:15:24,686 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@35e136ad, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-24T22:15:24,686 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request bd6592c37bb2,37701,-1 for getting cluster id 2024-11-24T22:15:24,686 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-24T22:15:24,687 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '3c461ac1-06fb-4739-8702-6368e685b03c' 2024-11-24T22:15:24,687 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-24T22:15:24,687 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "3c461ac1-06fb-4739-8702-6368e685b03c" 2024-11-24T22:15:24,687 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@62ade0cd, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-24T22:15:24,687 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [bd6592c37bb2,37701,-1] 2024-11-24T22:15:24,687 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-24T22:15:24,688 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-24T22:15:24,688 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:33982, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-24T22:15:24,689 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@da31f08, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-24T22:15:24,689 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@7f004f7c, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-24T22:15:24,689 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request bd6592c37bb2,37701,-1 for getting cluster id 2024-11-24T22:15:24,689 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-24T22:15:24,690 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '3c461ac1-06fb-4739-8702-6368e685b03c' 2024-11-24T22:15:24,690 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-24T22:15:24,690 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "3c461ac1-06fb-4739-8702-6368e685b03c" 2024-11-24T22:15:24,690 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@199d0784, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-24T22:15:24,690 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [bd6592c37bb2,37701,-1] 2024-11-24T22:15:24,690 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-24T22:15:24,690 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-24T22:15:24,691 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:34006, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-24T22:15:24,691 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@1f8b9a0d, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-24T22:15:24,692 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@35aa2c8b, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-24T22:15:24,692 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request bd6592c37bb2,37701,-1 for getting cluster id 2024-11-24T22:15:24,692 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-24T22:15:24,693 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '3c461ac1-06fb-4739-8702-6368e685b03c' 2024-11-24T22:15:24,693 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-24T22:15:24,693 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "3c461ac1-06fb-4739-8702-6368e685b03c" 2024-11-24T22:15:24,693 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@38f97bf1, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-24T22:15:24,693 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [bd6592c37bb2,37701,-1] 2024-11-24T22:15:24,693 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-24T22:15:24,693 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-24T22:15:24,694 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:34022, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-24T22:15:24,694 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@1bcb57e8, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-24T22:15:24,694 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@53f9edfd, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-24T22:15:24,694 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request bd6592c37bb2,37701,-1 for getting cluster id 2024-11-24T22:15:24,695 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-24T22:15:24,695 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '3c461ac1-06fb-4739-8702-6368e685b03c' 2024-11-24T22:15:24,695 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-24T22:15:24,695 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "3c461ac1-06fb-4739-8702-6368e685b03c" 2024-11-24T22:15:24,695 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@2fe9d36d, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-24T22:15:24,695 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [bd6592c37bb2,37701,-1] 2024-11-24T22:15:24,695 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-24T22:15:24,696 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-24T22:15:24,696 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:34042, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-24T22:15:24,697 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@14005ae9, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-24T22:15:24,697 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@42125f21, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-24T22:15:24,697 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request bd6592c37bb2,37701,-1 for getting cluster id 2024-11-24T22:15:24,697 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-24T22:15:24,698 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '3c461ac1-06fb-4739-8702-6368e685b03c' 2024-11-24T22:15:24,698 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-24T22:15:24,698 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "3c461ac1-06fb-4739-8702-6368e685b03c" 2024-11-24T22:15:24,698 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@442bce13, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-24T22:15:24,698 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [bd6592c37bb2,37701,-1] 2024-11-24T22:15:24,698 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-24T22:15:24,698 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-24T22:15:24,699 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:34072, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-24T22:15:24,699 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@7c959870, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-24T22:15:24,699 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@64b126a0, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-24T22:15:24,699 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request bd6592c37bb2,37701,-1 for getting cluster id 2024-11-24T22:15:24,700 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-24T22:15:24,700 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '3c461ac1-06fb-4739-8702-6368e685b03c' 2024-11-24T22:15:24,700 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-24T22:15:24,700 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "3c461ac1-06fb-4739-8702-6368e685b03c" 2024-11-24T22:15:24,700 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@60a66b37, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-24T22:15:24,700 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [bd6592c37bb2,37701,-1] 2024-11-24T22:15:24,700 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-24T22:15:24,701 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-24T22:15:24,701 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:34090, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-24T22:15:24,702 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@3424f2d9, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-24T22:15:24,702 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@4fa71035, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-24T22:15:24,702 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request bd6592c37bb2,37701,-1 for getting cluster id 2024-11-24T22:15:24,702 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-24T22:15:24,703 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '3c461ac1-06fb-4739-8702-6368e685b03c' 2024-11-24T22:15:24,703 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-24T22:15:24,703 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "3c461ac1-06fb-4739-8702-6368e685b03c" 2024-11-24T22:15:24,703 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@603d2548, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-24T22:15:24,703 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [bd6592c37bb2,37701,-1] 2024-11-24T22:15:24,703 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-24T22:15:24,703 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-24T22:15:24,704 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:34098, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-24T22:15:24,704 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@4efeb286, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-24T22:15:24,705 DEBUG [Thread-1614 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-24T22:15:24,705 DEBUG [Thread-1612 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-24T22:15:24,705 DEBUG [Thread-1608 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-24T22:15:24,705 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37701 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-24T22:15:24,705 DEBUG [Thread-1599 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-24T22:15:24,705 DEBUG [Thread-1606 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-24T22:15:24,705 DEBUG [Thread-1597 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-24T22:15:24,706 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=bd6592c37bb2,40087,1732486423713, seqNum=-1] 2024-11-24T22:15:24,706 DEBUG [Thread-1595 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-24T22:15:24,706 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=bd6592c37bb2,40087,1732486423713, seqNum=-1] 2024-11-24T22:15:24,706 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=bd6592c37bb2,40087,1732486423713, seqNum=-1] 2024-11-24T22:15:24,706 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37701 {}] procedure2.ProcedureExecutor(1139): Stored pid=97, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=97, table=TestAcidGuarantees 2024-11-24T22:15:24,706 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-24T22:15:24,706 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=bd6592c37bb2,40087,1732486423713, seqNum=-1] 2024-11-24T22:15:24,706 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-24T22:15:24,706 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-24T22:15:24,706 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-24T22:15:24,707 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=bd6592c37bb2,40087,1732486423713, seqNum=-1] 2024-11-24T22:15:24,707 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-24T22:15:24,707 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=bd6592c37bb2,40087,1732486423713, seqNum=-1] 2024-11-24T22:15:24,707 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-24T22:15:24,707 DEBUG [Thread-1601 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-24T22:15:24,707 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37701 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=97 2024-11-24T22:15:24,707 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=bd6592c37bb2,40087,1732486423713, seqNum=-1] 2024-11-24T22:15:24,707 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-24T22:15:24,708 DEBUG [Thread-1603 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-24T22:15:24,708 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:32776, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-24T22:15:24,708 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:32788, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-24T22:15:24,708 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:32774, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-24T22:15:24,708 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:32804, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-24T22:15:24,708 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:32802, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-24T22:15:24,708 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:32810, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-24T22:15:24,708 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=bd6592c37bb2,40087,1732486423713, seqNum=-1] 2024-11-24T22:15:24,708 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-24T22:15:24,709 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=97, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=97, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-24T22:15:24,709 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:32832, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-24T22:15:24,709 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:32822, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-24T22:15:24,709 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=97, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=97, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-24T22:15:24,709 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=98, ppid=97, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-24T22:15:24,710 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=bd6592c37bb2,40087,1732486423713, seqNum=-1] 2024-11-24T22:15:24,710 DEBUG [Thread-1610 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-24T22:15:24,710 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-24T22:15:24,711 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='', locateType=CURRENT is [region=TestAcidGuarantees,,1732486521610.94f3ccf9f8a638fa974e1f69627832ef., hostname=bd6592c37bb2,40087,1732486423713, seqNum=5] 2024-11-24T22:15:24,711 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=bd6592c37bb2,40087,1732486423713, seqNum=-1] 2024-11-24T22:15:24,711 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-24T22:15:24,712 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='', locateType=CURRENT is [region=TestAcidGuarantees,,1732486521610.94f3ccf9f8a638fa974e1f69627832ef., hostname=bd6592c37bb2,40087,1732486423713, seqNum=5] 2024-11-24T22:15:24,712 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:32846, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-24T22:15:24,712 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:32862, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-24T22:15:24,712 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='', locateType=CURRENT is [region=TestAcidGuarantees,,1732486521610.94f3ccf9f8a638fa974e1f69627832ef., hostname=bd6592c37bb2,40087,1732486423713, seqNum=5] 2024-11-24T22:15:24,713 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='', locateType=CURRENT is [region=TestAcidGuarantees,,1732486521610.94f3ccf9f8a638fa974e1f69627832ef., hostname=bd6592c37bb2,40087,1732486423713, seqNum=5] 2024-11-24T22:15:24,713 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_0', locateType=CURRENT is [region=TestAcidGuarantees,,1732486521610.94f3ccf9f8a638fa974e1f69627832ef., hostname=bd6592c37bb2,40087,1732486423713, seqNum=5] 2024-11-24T22:15:24,714 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_0', locateType=CURRENT is [region=TestAcidGuarantees,,1732486521610.94f3ccf9f8a638fa974e1f69627832ef., hostname=bd6592c37bb2,40087,1732486423713, seqNum=5] 2024-11-24T22:15:24,715 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_2', locateType=CURRENT is [region=TestAcidGuarantees,,1732486521610.94f3ccf9f8a638fa974e1f69627832ef., hostname=bd6592c37bb2,40087,1732486423713, seqNum=5] 2024-11-24T22:15:24,715 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='', locateType=CURRENT is [region=TestAcidGuarantees,,1732486521610.94f3ccf9f8a638fa974e1f69627832ef., hostname=bd6592c37bb2,40087,1732486423713, seqNum=5] 2024-11-24T22:15:24,715 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_0', locateType=CURRENT is [region=TestAcidGuarantees,,1732486521610.94f3ccf9f8a638fa974e1f69627832ef., hostname=bd6592c37bb2,40087,1732486423713, seqNum=5] 2024-11-24T22:15:24,717 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_2', locateType=CURRENT is [region=TestAcidGuarantees,,1732486521610.94f3ccf9f8a638fa974e1f69627832ef., hostname=bd6592c37bb2,40087,1732486423713, seqNum=5] 2024-11-24T22:15:24,730 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40087 {}] regionserver.HRegion(8855): Flush requested on 94f3ccf9f8a638fa974e1f69627832ef 2024-11-24T22:15:24,730 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 94f3ccf9f8a638fa974e1f69627832ef 3/3 column families, dataSize=53.67 KB heapSize=160.13 KB 2024-11-24T22:15:24,731 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 94f3ccf9f8a638fa974e1f69627832ef, store=A 2024-11-24T22:15:24,731 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-24T22:15:24,731 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 94f3ccf9f8a638fa974e1f69627832ef, store=B 2024-11-24T22:15:24,731 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-24T22:15:24,731 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 94f3ccf9f8a638fa974e1f69627832ef, store=C 2024-11-24T22:15:24,731 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-24T22:15:24,770 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241124ad2a51c8febb43ad978a48d99b3b0548_94f3ccf9f8a638fa974e1f69627832ef is 50, key is test_row_0/A:col10/1732486524716/Put/seqid=0 2024-11-24T22:15:24,774 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45929 is added to blk_1073742177_1353 (size=12154) 2024-11-24T22:15:24,811 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37701 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=97 2024-11-24T22:15:24,812 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40087 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94f3ccf9f8a638fa974e1f69627832ef, server=bd6592c37bb2,40087,1732486423713 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-24T22:15:24,813 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40087 {}] ipc.CallRunner(138): callId: 6 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:32774 deadline: 1732486584801, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94f3ccf9f8a638fa974e1f69627832ef, server=bd6592c37bb2,40087,1732486423713 2024-11-24T22:15:24,813 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40087 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94f3ccf9f8a638fa974e1f69627832ef, server=bd6592c37bb2,40087,1732486423713 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-24T22:15:24,813 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732486521610.94f3ccf9f8a638fa974e1f69627832ef., hostname=bd6592c37bb2,40087,1732486423713, seqNum=5 , the old value is region=TestAcidGuarantees,,1732486521610.94f3ccf9f8a638fa974e1f69627832ef., hostname=bd6592c37bb2,40087,1732486423713, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94f3ccf9f8a638fa974e1f69627832ef, server=bd6592c37bb2,40087,1732486423713 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T22:15:24,813 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40087 {}] ipc.CallRunner(138): callId: 6 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:32832 deadline: 1732486584801, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94f3ccf9f8a638fa974e1f69627832ef, server=bd6592c37bb2,40087,1732486423713 2024-11-24T22:15:24,813 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732486521610.94f3ccf9f8a638fa974e1f69627832ef., hostname=bd6592c37bb2,40087,1732486423713, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94f3ccf9f8a638fa974e1f69627832ef, server=bd6592c37bb2,40087,1732486423713 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T22:15:24,813 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732486521610.94f3ccf9f8a638fa974e1f69627832ef., hostname=bd6592c37bb2,40087,1732486423713, seqNum=5 because the exception is null or not the one we care about 2024-11-24T22:15:24,813 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732486521610.94f3ccf9f8a638fa974e1f69627832ef., hostname=bd6592c37bb2,40087,1732486423713, seqNum=5 , the old value is region=TestAcidGuarantees,,1732486521610.94f3ccf9f8a638fa974e1f69627832ef., hostname=bd6592c37bb2,40087,1732486423713, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94f3ccf9f8a638fa974e1f69627832ef, server=bd6592c37bb2,40087,1732486423713 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T22:15:24,813 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732486521610.94f3ccf9f8a638fa974e1f69627832ef., hostname=bd6592c37bb2,40087,1732486423713, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94f3ccf9f8a638fa974e1f69627832ef, server=bd6592c37bb2,40087,1732486423713 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T22:15:24,813 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732486521610.94f3ccf9f8a638fa974e1f69627832ef., hostname=bd6592c37bb2,40087,1732486423713, seqNum=5 because the exception is null or not the one we care about 2024-11-24T22:15:24,814 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40087 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94f3ccf9f8a638fa974e1f69627832ef, server=bd6592c37bb2,40087,1732486423713 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-24T22:15:24,814 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40087 {}] ipc.CallRunner(138): callId: 6 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:32822 deadline: 1732486584803, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94f3ccf9f8a638fa974e1f69627832ef, server=bd6592c37bb2,40087,1732486423713 2024-11-24T22:15:24,814 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40087 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94f3ccf9f8a638fa974e1f69627832ef, server=bd6592c37bb2,40087,1732486423713 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-24T22:15:24,814 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732486521610.94f3ccf9f8a638fa974e1f69627832ef., hostname=bd6592c37bb2,40087,1732486423713, seqNum=5 , the old value is region=TestAcidGuarantees,,1732486521610.94f3ccf9f8a638fa974e1f69627832ef., hostname=bd6592c37bb2,40087,1732486423713, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94f3ccf9f8a638fa974e1f69627832ef, server=bd6592c37bb2,40087,1732486423713 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T22:15:24,814 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732486521610.94f3ccf9f8a638fa974e1f69627832ef., hostname=bd6592c37bb2,40087,1732486423713, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94f3ccf9f8a638fa974e1f69627832ef, server=bd6592c37bb2,40087,1732486423713 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T22:15:24,814 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40087 {}] ipc.CallRunner(138): callId: 6 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:32846 deadline: 1732486584803, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94f3ccf9f8a638fa974e1f69627832ef, server=bd6592c37bb2,40087,1732486423713 2024-11-24T22:15:24,814 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732486521610.94f3ccf9f8a638fa974e1f69627832ef., hostname=bd6592c37bb2,40087,1732486423713, seqNum=5 because the exception is null or not the one we care about 2024-11-24T22:15:24,814 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732486521610.94f3ccf9f8a638fa974e1f69627832ef., hostname=bd6592c37bb2,40087,1732486423713, seqNum=5 , the old value is region=TestAcidGuarantees,,1732486521610.94f3ccf9f8a638fa974e1f69627832ef., hostname=bd6592c37bb2,40087,1732486423713, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94f3ccf9f8a638fa974e1f69627832ef, server=bd6592c37bb2,40087,1732486423713 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T22:15:24,814 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732486521610.94f3ccf9f8a638fa974e1f69627832ef., hostname=bd6592c37bb2,40087,1732486423713, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94f3ccf9f8a638fa974e1f69627832ef, server=bd6592c37bb2,40087,1732486423713 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T22:15:24,814 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732486521610.94f3ccf9f8a638fa974e1f69627832ef., hostname=bd6592c37bb2,40087,1732486423713, seqNum=5 because the exception is null or not the one we care about 2024-11-24T22:15:24,815 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40087 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94f3ccf9f8a638fa974e1f69627832ef, server=bd6592c37bb2,40087,1732486423713 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-24T22:15:24,816 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40087 {}] ipc.CallRunner(138): callId: 7 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:32810 deadline: 1732486584804, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94f3ccf9f8a638fa974e1f69627832ef, server=bd6592c37bb2,40087,1732486423713 2024-11-24T22:15:24,816 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732486521610.94f3ccf9f8a638fa974e1f69627832ef., hostname=bd6592c37bb2,40087,1732486423713, seqNum=5 , the old value is region=TestAcidGuarantees,,1732486521610.94f3ccf9f8a638fa974e1f69627832ef., hostname=bd6592c37bb2,40087,1732486423713, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94f3ccf9f8a638fa974e1f69627832ef, server=bd6592c37bb2,40087,1732486423713 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T22:15:24,816 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732486521610.94f3ccf9f8a638fa974e1f69627832ef., hostname=bd6592c37bb2,40087,1732486423713, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94f3ccf9f8a638fa974e1f69627832ef, server=bd6592c37bb2,40087,1732486423713 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T22:15:24,816 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732486521610.94f3ccf9f8a638fa974e1f69627832ef., hostname=bd6592c37bb2,40087,1732486423713, seqNum=5 because the exception is null or not the one we care about 2024-11-24T22:15:24,864 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=40087 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=98 2024-11-24T22:15:24,864 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bd6592c37bb2:0-1 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732486521610.94f3ccf9f8a638fa974e1f69627832ef. 2024-11-24T22:15:24,865 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bd6592c37bb2:0-1 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732486521610.94f3ccf9f8a638fa974e1f69627832ef. as already flushing 2024-11-24T22:15:24,865 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bd6592c37bb2:0-1 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732486521610.94f3ccf9f8a638fa974e1f69627832ef. 2024-11-24T22:15:24,865 ERROR [RS_FLUSH_OPERATIONS-regionserver/bd6592c37bb2:0-1 {event_type=RS_FLUSH_REGIONS, pid=98}] handler.RSProcedureHandler(58): pid=98 java.io.IOException: Unable to complete flush {ENCODED => 94f3ccf9f8a638fa974e1f69627832ef, NAME => 'TestAcidGuarantees,,1732486521610.94f3ccf9f8a638fa974e1f69627832ef.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T22:15:24,865 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bd6592c37bb2:0-1 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=98 java.io.IOException: Unable to complete flush {ENCODED => 94f3ccf9f8a638fa974e1f69627832ef, NAME => 'TestAcidGuarantees,,1732486521610.94f3ccf9f8a638fa974e1f69627832ef.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T22:15:24,865 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37701 {}] master.HMaster(4177): Remote procedure failed, pid=98 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 94f3ccf9f8a638fa974e1f69627832ef, NAME => 'TestAcidGuarantees,,1732486521610.94f3ccf9f8a638fa974e1f69627832ef.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 94f3ccf9f8a638fa974e1f69627832ef, NAME => 'TestAcidGuarantees,,1732486521610.94f3ccf9f8a638fa974e1f69627832ef.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T22:15:24,925 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40087 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94f3ccf9f8a638fa974e1f69627832ef, server=bd6592c37bb2,40087,1732486423713 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-24T22:15:24,925 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40087 {}] ipc.CallRunner(138): callId: 7 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:32774 deadline: 1732486584921, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94f3ccf9f8a638fa974e1f69627832ef, server=bd6592c37bb2,40087,1732486423713 2024-11-24T22:15:24,925 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40087 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94f3ccf9f8a638fa974e1f69627832ef, server=bd6592c37bb2,40087,1732486423713 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-24T22:15:24,926 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40087 {}] ipc.CallRunner(138): callId: 7 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:32832 deadline: 1732486584921, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94f3ccf9f8a638fa974e1f69627832ef, server=bd6592c37bb2,40087,1732486423713 2024-11-24T22:15:24,926 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732486521610.94f3ccf9f8a638fa974e1f69627832ef., hostname=bd6592c37bb2,40087,1732486423713, seqNum=5 , the old value is region=TestAcidGuarantees,,1732486521610.94f3ccf9f8a638fa974e1f69627832ef., hostname=bd6592c37bb2,40087,1732486423713, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94f3ccf9f8a638fa974e1f69627832ef, server=bd6592c37bb2,40087,1732486423713 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T22:15:24,926 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732486521610.94f3ccf9f8a638fa974e1f69627832ef., hostname=bd6592c37bb2,40087,1732486423713, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94f3ccf9f8a638fa974e1f69627832ef, server=bd6592c37bb2,40087,1732486423713 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T22:15:24,926 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732486521610.94f3ccf9f8a638fa974e1f69627832ef., hostname=bd6592c37bb2,40087,1732486423713, seqNum=5 because the exception is null or not the one we care about 2024-11-24T22:15:24,926 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40087 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94f3ccf9f8a638fa974e1f69627832ef, server=bd6592c37bb2,40087,1732486423713 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-24T22:15:24,926 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40087 {}] ipc.CallRunner(138): callId: 7 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:32822 deadline: 1732486584921, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94f3ccf9f8a638fa974e1f69627832ef, server=bd6592c37bb2,40087,1732486423713 2024-11-24T22:15:24,926 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732486521610.94f3ccf9f8a638fa974e1f69627832ef., hostname=bd6592c37bb2,40087,1732486423713, seqNum=5 , the old value is region=TestAcidGuarantees,,1732486521610.94f3ccf9f8a638fa974e1f69627832ef., hostname=bd6592c37bb2,40087,1732486423713, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94f3ccf9f8a638fa974e1f69627832ef, server=bd6592c37bb2,40087,1732486423713 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T22:15:24,926 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732486521610.94f3ccf9f8a638fa974e1f69627832ef., hostname=bd6592c37bb2,40087,1732486423713, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94f3ccf9f8a638fa974e1f69627832ef, server=bd6592c37bb2,40087,1732486423713 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T22:15:24,926 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732486521610.94f3ccf9f8a638fa974e1f69627832ef., hostname=bd6592c37bb2,40087,1732486423713, seqNum=5 because the exception is null or not the one we care about 2024-11-24T22:15:24,926 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40087 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94f3ccf9f8a638fa974e1f69627832ef, server=bd6592c37bb2,40087,1732486423713 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-24T22:15:24,926 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732486521610.94f3ccf9f8a638fa974e1f69627832ef., hostname=bd6592c37bb2,40087,1732486423713, seqNum=5 , the old value is region=TestAcidGuarantees,,1732486521610.94f3ccf9f8a638fa974e1f69627832ef., hostname=bd6592c37bb2,40087,1732486423713, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94f3ccf9f8a638fa974e1f69627832ef, server=bd6592c37bb2,40087,1732486423713 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T22:15:24,926 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732486521610.94f3ccf9f8a638fa974e1f69627832ef., hostname=bd6592c37bb2,40087,1732486423713, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94f3ccf9f8a638fa974e1f69627832ef, server=bd6592c37bb2,40087,1732486423713 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T22:15:24,926 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40087 {}] ipc.CallRunner(138): callId: 7 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:32846 deadline: 1732486584921, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94f3ccf9f8a638fa974e1f69627832ef, server=bd6592c37bb2,40087,1732486423713 2024-11-24T22:15:24,926 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732486521610.94f3ccf9f8a638fa974e1f69627832ef., hostname=bd6592c37bb2,40087,1732486423713, seqNum=5 because the exception is null or not the one we care about 2024-11-24T22:15:24,927 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40087 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94f3ccf9f8a638fa974e1f69627832ef, server=bd6592c37bb2,40087,1732486423713 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-24T22:15:24,927 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732486521610.94f3ccf9f8a638fa974e1f69627832ef., hostname=bd6592c37bb2,40087,1732486423713, seqNum=5 , the old value is region=TestAcidGuarantees,,1732486521610.94f3ccf9f8a638fa974e1f69627832ef., hostname=bd6592c37bb2,40087,1732486423713, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94f3ccf9f8a638fa974e1f69627832ef, server=bd6592c37bb2,40087,1732486423713 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T22:15:24,927 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732486521610.94f3ccf9f8a638fa974e1f69627832ef., hostname=bd6592c37bb2,40087,1732486423713, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94f3ccf9f8a638fa974e1f69627832ef, server=bd6592c37bb2,40087,1732486423713 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T22:15:24,927 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40087 {}] ipc.CallRunner(138): callId: 8 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:32810 deadline: 1732486584921, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94f3ccf9f8a638fa974e1f69627832ef, server=bd6592c37bb2,40087,1732486423713 2024-11-24T22:15:24,927 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732486521610.94f3ccf9f8a638fa974e1f69627832ef., hostname=bd6592c37bb2,40087,1732486423713, seqNum=5 because the exception is null or not the one we care about 2024-11-24T22:15:24,927 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732486521610.94f3ccf9f8a638fa974e1f69627832ef., hostname=bd6592c37bb2,40087,1732486423713, seqNum=5 , the old value is region=TestAcidGuarantees,,1732486521610.94f3ccf9f8a638fa974e1f69627832ef., hostname=bd6592c37bb2,40087,1732486423713, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94f3ccf9f8a638fa974e1f69627832ef, server=bd6592c37bb2,40087,1732486423713 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T22:15:24,927 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732486521610.94f3ccf9f8a638fa974e1f69627832ef., hostname=bd6592c37bb2,40087,1732486423713, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94f3ccf9f8a638fa974e1f69627832ef, server=bd6592c37bb2,40087,1732486423713 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T22:15:24,927 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732486521610.94f3ccf9f8a638fa974e1f69627832ef., hostname=bd6592c37bb2,40087,1732486423713, seqNum=5 because the exception is null or not the one we care about 2024-11-24T22:15:25,017 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=40087 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=98 2024-11-24T22:15:25,017 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bd6592c37bb2:0-2 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732486521610.94f3ccf9f8a638fa974e1f69627832ef. 2024-11-24T22:15:25,017 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bd6592c37bb2:0-2 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732486521610.94f3ccf9f8a638fa974e1f69627832ef. as already flushing 2024-11-24T22:15:25,017 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bd6592c37bb2:0-2 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732486521610.94f3ccf9f8a638fa974e1f69627832ef. 2024-11-24T22:15:25,018 ERROR [RS_FLUSH_OPERATIONS-regionserver/bd6592c37bb2:0-2 {event_type=RS_FLUSH_REGIONS, pid=98}] handler.RSProcedureHandler(58): pid=98 java.io.IOException: Unable to complete flush {ENCODED => 94f3ccf9f8a638fa974e1f69627832ef, NAME => 'TestAcidGuarantees,,1732486521610.94f3ccf9f8a638fa974e1f69627832ef.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T22:15:25,018 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bd6592c37bb2:0-2 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=98 java.io.IOException: Unable to complete flush {ENCODED => 94f3ccf9f8a638fa974e1f69627832ef, NAME => 'TestAcidGuarantees,,1732486521610.94f3ccf9f8a638fa974e1f69627832ef.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T22:15:25,018 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37701 {}] master.HMaster(4177): Remote procedure failed, pid=98 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 94f3ccf9f8a638fa974e1f69627832ef, NAME => 'TestAcidGuarantees,,1732486521610.94f3ccf9f8a638fa974e1f69627832ef.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 94f3ccf9f8a638fa974e1f69627832ef, NAME => 'TestAcidGuarantees,,1732486521610.94f3ccf9f8a638fa974e1f69627832ef.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T22:15:25,020 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37701 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=97 2024-11-24T22:15:25,136 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40087 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94f3ccf9f8a638fa974e1f69627832ef, server=bd6592c37bb2,40087,1732486423713 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-24T22:15:25,136 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40087 {}] ipc.CallRunner(138): callId: 8 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:32774 deadline: 1732486585131, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94f3ccf9f8a638fa974e1f69627832ef, server=bd6592c37bb2,40087,1732486423713 2024-11-24T22:15:25,136 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40087 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94f3ccf9f8a638fa974e1f69627832ef, server=bd6592c37bb2,40087,1732486423713 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-24T22:15:25,136 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732486521610.94f3ccf9f8a638fa974e1f69627832ef., hostname=bd6592c37bb2,40087,1732486423713, seqNum=5 , the old value is region=TestAcidGuarantees,,1732486521610.94f3ccf9f8a638fa974e1f69627832ef., hostname=bd6592c37bb2,40087,1732486423713, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94f3ccf9f8a638fa974e1f69627832ef, server=bd6592c37bb2,40087,1732486423713 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T22:15:25,136 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732486521610.94f3ccf9f8a638fa974e1f69627832ef., hostname=bd6592c37bb2,40087,1732486423713, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94f3ccf9f8a638fa974e1f69627832ef, server=bd6592c37bb2,40087,1732486423713 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T22:15:25,136 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40087 {}] ipc.CallRunner(138): callId: 8 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:32832 deadline: 1732486585131, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94f3ccf9f8a638fa974e1f69627832ef, server=bd6592c37bb2,40087,1732486423713 2024-11-24T22:15:25,136 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732486521610.94f3ccf9f8a638fa974e1f69627832ef., hostname=bd6592c37bb2,40087,1732486423713, seqNum=5 because the exception is null or not the one we care about 2024-11-24T22:15:25,137 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40087 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94f3ccf9f8a638fa974e1f69627832ef, server=bd6592c37bb2,40087,1732486423713 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-24T22:15:25,137 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40087 {}] ipc.CallRunner(138): callId: 8 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:32822 deadline: 1732486585132, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94f3ccf9f8a638fa974e1f69627832ef, server=bd6592c37bb2,40087,1732486423713 2024-11-24T22:15:25,137 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732486521610.94f3ccf9f8a638fa974e1f69627832ef., hostname=bd6592c37bb2,40087,1732486423713, seqNum=5 , the old value is region=TestAcidGuarantees,,1732486521610.94f3ccf9f8a638fa974e1f69627832ef., hostname=bd6592c37bb2,40087,1732486423713, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94f3ccf9f8a638fa974e1f69627832ef, server=bd6592c37bb2,40087,1732486423713 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T22:15:25,137 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732486521610.94f3ccf9f8a638fa974e1f69627832ef., hostname=bd6592c37bb2,40087,1732486423713, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94f3ccf9f8a638fa974e1f69627832ef, server=bd6592c37bb2,40087,1732486423713 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T22:15:25,137 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732486521610.94f3ccf9f8a638fa974e1f69627832ef., hostname=bd6592c37bb2,40087,1732486423713, seqNum=5 because the exception is null or not the one we care about 2024-11-24T22:15:25,137 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40087 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94f3ccf9f8a638fa974e1f69627832ef, server=bd6592c37bb2,40087,1732486423713 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-24T22:15:25,137 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40087 {}] ipc.CallRunner(138): callId: 8 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:32846 deadline: 1732486585132, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94f3ccf9f8a638fa974e1f69627832ef, server=bd6592c37bb2,40087,1732486423713 2024-11-24T22:15:25,137 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732486521610.94f3ccf9f8a638fa974e1f69627832ef., hostname=bd6592c37bb2,40087,1732486423713, seqNum=5 , the old value is region=TestAcidGuarantees,,1732486521610.94f3ccf9f8a638fa974e1f69627832ef., hostname=bd6592c37bb2,40087,1732486423713, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94f3ccf9f8a638fa974e1f69627832ef, server=bd6592c37bb2,40087,1732486423713 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T22:15:25,137 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732486521610.94f3ccf9f8a638fa974e1f69627832ef., hostname=bd6592c37bb2,40087,1732486423713, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94f3ccf9f8a638fa974e1f69627832ef, server=bd6592c37bb2,40087,1732486423713 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T22:15:25,137 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732486521610.94f3ccf9f8a638fa974e1f69627832ef., hostname=bd6592c37bb2,40087,1732486423713, seqNum=5 because the exception is null or not the one we care about 2024-11-24T22:15:25,137 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40087 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94f3ccf9f8a638fa974e1f69627832ef, server=bd6592c37bb2,40087,1732486423713 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-24T22:15:25,137 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732486521610.94f3ccf9f8a638fa974e1f69627832ef., hostname=bd6592c37bb2,40087,1732486423713, seqNum=5 , the old value is region=TestAcidGuarantees,,1732486521610.94f3ccf9f8a638fa974e1f69627832ef., hostname=bd6592c37bb2,40087,1732486423713, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94f3ccf9f8a638fa974e1f69627832ef, server=bd6592c37bb2,40087,1732486423713 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T22:15:25,137 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40087 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:32810 deadline: 1732486585132, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94f3ccf9f8a638fa974e1f69627832ef, server=bd6592c37bb2,40087,1732486423713 2024-11-24T22:15:25,137 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732486521610.94f3ccf9f8a638fa974e1f69627832ef., hostname=bd6592c37bb2,40087,1732486423713, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94f3ccf9f8a638fa974e1f69627832ef, server=bd6592c37bb2,40087,1732486423713 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T22:15:25,137 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732486521610.94f3ccf9f8a638fa974e1f69627832ef., hostname=bd6592c37bb2,40087,1732486423713, seqNum=5 because the exception is null or not the one we care about 2024-11-24T22:15:25,138 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732486521610.94f3ccf9f8a638fa974e1f69627832ef., hostname=bd6592c37bb2,40087,1732486423713, seqNum=5 , the old value is region=TestAcidGuarantees,,1732486521610.94f3ccf9f8a638fa974e1f69627832ef., hostname=bd6592c37bb2,40087,1732486423713, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94f3ccf9f8a638fa974e1f69627832ef, server=bd6592c37bb2,40087,1732486423713 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T22:15:25,138 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732486521610.94f3ccf9f8a638fa974e1f69627832ef., hostname=bd6592c37bb2,40087,1732486423713, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94f3ccf9f8a638fa974e1f69627832ef, server=bd6592c37bb2,40087,1732486423713 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T22:15:25,138 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732486521610.94f3ccf9f8a638fa974e1f69627832ef., hostname=bd6592c37bb2,40087,1732486423713, seqNum=5 because the exception is null or not the one we care about 2024-11-24T22:15:25,170 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=40087 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=98 2024-11-24T22:15:25,170 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bd6592c37bb2:0-0 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732486521610.94f3ccf9f8a638fa974e1f69627832ef. 2024-11-24T22:15:25,170 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bd6592c37bb2:0-0 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732486521610.94f3ccf9f8a638fa974e1f69627832ef. as already flushing 2024-11-24T22:15:25,170 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bd6592c37bb2:0-0 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732486521610.94f3ccf9f8a638fa974e1f69627832ef. 2024-11-24T22:15:25,170 ERROR [RS_FLUSH_OPERATIONS-regionserver/bd6592c37bb2:0-0 {event_type=RS_FLUSH_REGIONS, pid=98}] handler.RSProcedureHandler(58): pid=98 java.io.IOException: Unable to complete flush {ENCODED => 94f3ccf9f8a638fa974e1f69627832ef, NAME => 'TestAcidGuarantees,,1732486521610.94f3ccf9f8a638fa974e1f69627832ef.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T22:15:25,171 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bd6592c37bb2:0-0 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=98 java.io.IOException: Unable to complete flush {ENCODED => 94f3ccf9f8a638fa974e1f69627832ef, NAME => 'TestAcidGuarantees,,1732486521610.94f3ccf9f8a638fa974e1f69627832ef.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T22:15:25,171 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37701 {}] master.HMaster(4177): Remote procedure failed, pid=98 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 94f3ccf9f8a638fa974e1f69627832ef, NAME => 'TestAcidGuarantees,,1732486521610.94f3ccf9f8a638fa974e1f69627832ef.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 94f3ccf9f8a638fa974e1f69627832ef, NAME => 'TestAcidGuarantees,,1732486521610.94f3ccf9f8a638fa974e1f69627832ef.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T22:15:25,175 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T22:15:25,178 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241124ad2a51c8febb43ad978a48d99b3b0548_94f3ccf9f8a638fa974e1f69627832ef to hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241124ad2a51c8febb43ad978a48d99b3b0548_94f3ccf9f8a638fa974e1f69627832ef 2024-11-24T22:15:25,179 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/94f3ccf9f8a638fa974e1f69627832ef/.tmp/A/a8661fe5f09945f8996df4fc0affd5f9, store: [table=TestAcidGuarantees family=A region=94f3ccf9f8a638fa974e1f69627832ef] 2024-11-24T22:15:25,180 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/94f3ccf9f8a638fa974e1f69627832ef/.tmp/A/a8661fe5f09945f8996df4fc0affd5f9 is 175, key is test_row_0/A:col10/1732486524716/Put/seqid=0 2024-11-24T22:15:25,186 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45929 is added to blk_1073742178_1354 (size=30955) 2024-11-24T22:15:25,323 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=40087 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=98 2024-11-24T22:15:25,323 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bd6592c37bb2:0-1 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732486521610.94f3ccf9f8a638fa974e1f69627832ef. 2024-11-24T22:15:25,323 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bd6592c37bb2:0-1 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732486521610.94f3ccf9f8a638fa974e1f69627832ef. as already flushing 2024-11-24T22:15:25,323 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bd6592c37bb2:0-1 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732486521610.94f3ccf9f8a638fa974e1f69627832ef. 2024-11-24T22:15:25,323 ERROR [RS_FLUSH_OPERATIONS-regionserver/bd6592c37bb2:0-1 {event_type=RS_FLUSH_REGIONS, pid=98}] handler.RSProcedureHandler(58): pid=98 java.io.IOException: Unable to complete flush {ENCODED => 94f3ccf9f8a638fa974e1f69627832ef, NAME => 'TestAcidGuarantees,,1732486521610.94f3ccf9f8a638fa974e1f69627832ef.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T22:15:25,324 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bd6592c37bb2:0-1 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=98 java.io.IOException: Unable to complete flush {ENCODED => 94f3ccf9f8a638fa974e1f69627832ef, NAME => 'TestAcidGuarantees,,1732486521610.94f3ccf9f8a638fa974e1f69627832ef.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T22:15:25,324 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37701 {}] master.HMaster(4177): Remote procedure failed, pid=98 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 94f3ccf9f8a638fa974e1f69627832ef, NAME => 'TestAcidGuarantees,,1732486521610.94f3ccf9f8a638fa974e1f69627832ef.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 94f3ccf9f8a638fa974e1f69627832ef, NAME => 'TestAcidGuarantees,,1732486521610.94f3ccf9f8a638fa974e1f69627832ef.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T22:15:25,331 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37701 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=97 2024-11-24T22:15:25,447 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40087 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94f3ccf9f8a638fa974e1f69627832ef, server=bd6592c37bb2,40087,1732486423713 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-24T22:15:25,447 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40087 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:32774 deadline: 1732486585441, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94f3ccf9f8a638fa974e1f69627832ef, server=bd6592c37bb2,40087,1732486423713 2024-11-24T22:15:25,448 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40087 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94f3ccf9f8a638fa974e1f69627832ef, server=bd6592c37bb2,40087,1732486423713 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-24T22:15:25,448 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732486521610.94f3ccf9f8a638fa974e1f69627832ef., hostname=bd6592c37bb2,40087,1732486423713, seqNum=5 , the old value is region=TestAcidGuarantees,,1732486521610.94f3ccf9f8a638fa974e1f69627832ef., hostname=bd6592c37bb2,40087,1732486423713, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94f3ccf9f8a638fa974e1f69627832ef, server=bd6592c37bb2,40087,1732486423713 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T22:15:25,448 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732486521610.94f3ccf9f8a638fa974e1f69627832ef., hostname=bd6592c37bb2,40087,1732486423713, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94f3ccf9f8a638fa974e1f69627832ef, server=bd6592c37bb2,40087,1732486423713 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T22:15:25,448 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732486521610.94f3ccf9f8a638fa974e1f69627832ef., hostname=bd6592c37bb2,40087,1732486423713, seqNum=5 because the exception is null or not the one we care about 2024-11-24T22:15:25,448 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40087 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:32822 deadline: 1732486585441, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94f3ccf9f8a638fa974e1f69627832ef, server=bd6592c37bb2,40087,1732486423713 2024-11-24T22:15:25,448 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40087 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94f3ccf9f8a638fa974e1f69627832ef, server=bd6592c37bb2,40087,1732486423713 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-24T22:15:25,448 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732486521610.94f3ccf9f8a638fa974e1f69627832ef., hostname=bd6592c37bb2,40087,1732486423713, seqNum=5 , the old value is region=TestAcidGuarantees,,1732486521610.94f3ccf9f8a638fa974e1f69627832ef., hostname=bd6592c37bb2,40087,1732486423713, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94f3ccf9f8a638fa974e1f69627832ef, server=bd6592c37bb2,40087,1732486423713 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T22:15:25,448 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732486521610.94f3ccf9f8a638fa974e1f69627832ef., hostname=bd6592c37bb2,40087,1732486423713, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94f3ccf9f8a638fa974e1f69627832ef, server=bd6592c37bb2,40087,1732486423713 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T22:15:25,448 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40087 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:32846 deadline: 1732486585441, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94f3ccf9f8a638fa974e1f69627832ef, server=bd6592c37bb2,40087,1732486423713 2024-11-24T22:15:25,448 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732486521610.94f3ccf9f8a638fa974e1f69627832ef., hostname=bd6592c37bb2,40087,1732486423713, seqNum=5 because the exception is null or not the one we care about 2024-11-24T22:15:25,448 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40087 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94f3ccf9f8a638fa974e1f69627832ef, server=bd6592c37bb2,40087,1732486423713 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-24T22:15:25,448 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732486521610.94f3ccf9f8a638fa974e1f69627832ef., hostname=bd6592c37bb2,40087,1732486423713, seqNum=5 , the old value is region=TestAcidGuarantees,,1732486521610.94f3ccf9f8a638fa974e1f69627832ef., hostname=bd6592c37bb2,40087,1732486423713, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94f3ccf9f8a638fa974e1f69627832ef, server=bd6592c37bb2,40087,1732486423713 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T22:15:25,448 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40087 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:32810 deadline: 1732486585441, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94f3ccf9f8a638fa974e1f69627832ef, server=bd6592c37bb2,40087,1732486423713 2024-11-24T22:15:25,448 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732486521610.94f3ccf9f8a638fa974e1f69627832ef., hostname=bd6592c37bb2,40087,1732486423713, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94f3ccf9f8a638fa974e1f69627832ef, server=bd6592c37bb2,40087,1732486423713 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T22:15:25,449 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732486521610.94f3ccf9f8a638fa974e1f69627832ef., hostname=bd6592c37bb2,40087,1732486423713, seqNum=5 because the exception is null or not the one we care about 2024-11-24T22:15:25,449 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732486521610.94f3ccf9f8a638fa974e1f69627832ef., hostname=bd6592c37bb2,40087,1732486423713, seqNum=5 , the old value is region=TestAcidGuarantees,,1732486521610.94f3ccf9f8a638fa974e1f69627832ef., hostname=bd6592c37bb2,40087,1732486423713, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94f3ccf9f8a638fa974e1f69627832ef, server=bd6592c37bb2,40087,1732486423713 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T22:15:25,449 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732486521610.94f3ccf9f8a638fa974e1f69627832ef., hostname=bd6592c37bb2,40087,1732486423713, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94f3ccf9f8a638fa974e1f69627832ef, server=bd6592c37bb2,40087,1732486423713 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T22:15:25,449 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732486521610.94f3ccf9f8a638fa974e1f69627832ef., hostname=bd6592c37bb2,40087,1732486423713, seqNum=5 because the exception is null or not the one we care about 2024-11-24T22:15:25,460 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40087 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94f3ccf9f8a638fa974e1f69627832ef, server=bd6592c37bb2,40087,1732486423713 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-24T22:15:25,461 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40087 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:32832 deadline: 1732486585451, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94f3ccf9f8a638fa974e1f69627832ef, server=bd6592c37bb2,40087,1732486423713 2024-11-24T22:15:25,461 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732486521610.94f3ccf9f8a638fa974e1f69627832ef., hostname=bd6592c37bb2,40087,1732486423713, seqNum=5 , the old value is region=TestAcidGuarantees,,1732486521610.94f3ccf9f8a638fa974e1f69627832ef., hostname=bd6592c37bb2,40087,1732486423713, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94f3ccf9f8a638fa974e1f69627832ef, server=bd6592c37bb2,40087,1732486423713 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T22:15:25,461 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732486521610.94f3ccf9f8a638fa974e1f69627832ef., hostname=bd6592c37bb2,40087,1732486423713, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94f3ccf9f8a638fa974e1f69627832ef, server=bd6592c37bb2,40087,1732486423713 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T22:15:25,461 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732486521610.94f3ccf9f8a638fa974e1f69627832ef., hostname=bd6592c37bb2,40087,1732486423713, seqNum=5 because the exception is null or not the one we care about 2024-11-24T22:15:25,476 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=40087 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=98 2024-11-24T22:15:25,476 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bd6592c37bb2:0-2 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732486521610.94f3ccf9f8a638fa974e1f69627832ef. 2024-11-24T22:15:25,476 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bd6592c37bb2:0-2 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732486521610.94f3ccf9f8a638fa974e1f69627832ef. as already flushing 2024-11-24T22:15:25,476 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bd6592c37bb2:0-2 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732486521610.94f3ccf9f8a638fa974e1f69627832ef. 2024-11-24T22:15:25,476 ERROR [RS_FLUSH_OPERATIONS-regionserver/bd6592c37bb2:0-2 {event_type=RS_FLUSH_REGIONS, pid=98}] handler.RSProcedureHandler(58): pid=98 java.io.IOException: Unable to complete flush {ENCODED => 94f3ccf9f8a638fa974e1f69627832ef, NAME => 'TestAcidGuarantees,,1732486521610.94f3ccf9f8a638fa974e1f69627832ef.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T22:15:25,476 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bd6592c37bb2:0-2 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=98 java.io.IOException: Unable to complete flush {ENCODED => 94f3ccf9f8a638fa974e1f69627832ef, NAME => 'TestAcidGuarantees,,1732486521610.94f3ccf9f8a638fa974e1f69627832ef.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T22:15:25,477 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37701 {}] master.HMaster(4177): Remote procedure failed, pid=98 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 94f3ccf9f8a638fa974e1f69627832ef, NAME => 'TestAcidGuarantees,,1732486521610.94f3ccf9f8a638fa974e1f69627832ef.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 94f3ccf9f8a638fa974e1f69627832ef, NAME => 'TestAcidGuarantees,,1732486521610.94f3ccf9f8a638fa974e1f69627832ef.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T22:15:25,587 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=16, memsize=20.1 K, hasBloomFilter=true, into tmp file hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/94f3ccf9f8a638fa974e1f69627832ef/.tmp/A/a8661fe5f09945f8996df4fc0affd5f9 2024-11-24T22:15:25,612 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/94f3ccf9f8a638fa974e1f69627832ef/.tmp/B/cd94decc27924b9687ddf2ec811ae351 is 50, key is test_row_0/B:col10/1732486524716/Put/seqid=0 2024-11-24T22:15:25,616 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45929 is added to blk_1073742179_1355 (size=12001) 2024-11-24T22:15:25,629 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=40087 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=98 2024-11-24T22:15:25,629 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bd6592c37bb2:0-0 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732486521610.94f3ccf9f8a638fa974e1f69627832ef. 2024-11-24T22:15:25,629 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bd6592c37bb2:0-0 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732486521610.94f3ccf9f8a638fa974e1f69627832ef. as already flushing 2024-11-24T22:15:25,629 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bd6592c37bb2:0-0 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732486521610.94f3ccf9f8a638fa974e1f69627832ef. 2024-11-24T22:15:25,629 ERROR [RS_FLUSH_OPERATIONS-regionserver/bd6592c37bb2:0-0 {event_type=RS_FLUSH_REGIONS, pid=98}] handler.RSProcedureHandler(58): pid=98 java.io.IOException: Unable to complete flush {ENCODED => 94f3ccf9f8a638fa974e1f69627832ef, NAME => 'TestAcidGuarantees,,1732486521610.94f3ccf9f8a638fa974e1f69627832ef.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T22:15:25,629 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bd6592c37bb2:0-0 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=98 java.io.IOException: Unable to complete flush {ENCODED => 94f3ccf9f8a638fa974e1f69627832ef, NAME => 'TestAcidGuarantees,,1732486521610.94f3ccf9f8a638fa974e1f69627832ef.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T22:15:25,630 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37701 {}] master.HMaster(4177): Remote procedure failed, pid=98 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 94f3ccf9f8a638fa974e1f69627832ef, NAME => 'TestAcidGuarantees,,1732486521610.94f3ccf9f8a638fa974e1f69627832ef.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 94f3ccf9f8a638fa974e1f69627832ef, NAME => 'TestAcidGuarantees,,1732486521610.94f3ccf9f8a638fa974e1f69627832ef.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T22:15:25,781 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=40087 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=98 2024-11-24T22:15:25,782 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bd6592c37bb2:0-1 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732486521610.94f3ccf9f8a638fa974e1f69627832ef. 2024-11-24T22:15:25,782 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bd6592c37bb2:0-1 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732486521610.94f3ccf9f8a638fa974e1f69627832ef. as already flushing 2024-11-24T22:15:25,782 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bd6592c37bb2:0-1 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732486521610.94f3ccf9f8a638fa974e1f69627832ef. 2024-11-24T22:15:25,782 ERROR [RS_FLUSH_OPERATIONS-regionserver/bd6592c37bb2:0-1 {event_type=RS_FLUSH_REGIONS, pid=98}] handler.RSProcedureHandler(58): pid=98 java.io.IOException: Unable to complete flush {ENCODED => 94f3ccf9f8a638fa974e1f69627832ef, NAME => 'TestAcidGuarantees,,1732486521610.94f3ccf9f8a638fa974e1f69627832ef.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T22:15:25,782 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bd6592c37bb2:0-1 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=98 java.io.IOException: Unable to complete flush {ENCODED => 94f3ccf9f8a638fa974e1f69627832ef, NAME => 'TestAcidGuarantees,,1732486521610.94f3ccf9f8a638fa974e1f69627832ef.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T22:15:25,783 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37701 {}] master.HMaster(4177): Remote procedure failed, pid=98 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 94f3ccf9f8a638fa974e1f69627832ef, NAME => 'TestAcidGuarantees,,1732486521610.94f3ccf9f8a638fa974e1f69627832ef.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 94f3ccf9f8a638fa974e1f69627832ef, NAME => 'TestAcidGuarantees,,1732486521610.94f3ccf9f8a638fa974e1f69627832ef.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T22:15:25,841 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37701 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=97 2024-11-24T22:15:25,935 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=40087 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=98 2024-11-24T22:15:25,935 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bd6592c37bb2:0-2 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732486521610.94f3ccf9f8a638fa974e1f69627832ef. 2024-11-24T22:15:25,935 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bd6592c37bb2:0-2 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732486521610.94f3ccf9f8a638fa974e1f69627832ef. as already flushing 2024-11-24T22:15:25,935 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bd6592c37bb2:0-2 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732486521610.94f3ccf9f8a638fa974e1f69627832ef. 2024-11-24T22:15:25,935 ERROR [RS_FLUSH_OPERATIONS-regionserver/bd6592c37bb2:0-2 {event_type=RS_FLUSH_REGIONS, pid=98}] handler.RSProcedureHandler(58): pid=98 java.io.IOException: Unable to complete flush {ENCODED => 94f3ccf9f8a638fa974e1f69627832ef, NAME => 'TestAcidGuarantees,,1732486521610.94f3ccf9f8a638fa974e1f69627832ef.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T22:15:25,935 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bd6592c37bb2:0-2 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=98 java.io.IOException: Unable to complete flush {ENCODED => 94f3ccf9f8a638fa974e1f69627832ef, NAME => 'TestAcidGuarantees,,1732486521610.94f3ccf9f8a638fa974e1f69627832ef.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T22:15:25,936 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37701 {}] master.HMaster(4177): Remote procedure failed, pid=98 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 94f3ccf9f8a638fa974e1f69627832ef, NAME => 'TestAcidGuarantees,,1732486521610.94f3ccf9f8a638fa974e1f69627832ef.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 94f3ccf9f8a638fa974e1f69627832ef, NAME => 'TestAcidGuarantees,,1732486521610.94f3ccf9f8a638fa974e1f69627832ef.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T22:15:25,957 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40087 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94f3ccf9f8a638fa974e1f69627832ef, server=bd6592c37bb2,40087,1732486423713 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-24T22:15:25,958 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40087 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:32774 deadline: 1732486585950, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94f3ccf9f8a638fa974e1f69627832ef, server=bd6592c37bb2,40087,1732486423713 2024-11-24T22:15:25,958 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732486521610.94f3ccf9f8a638fa974e1f69627832ef., hostname=bd6592c37bb2,40087,1732486423713, seqNum=5 , the old value is region=TestAcidGuarantees,,1732486521610.94f3ccf9f8a638fa974e1f69627832ef., hostname=bd6592c37bb2,40087,1732486423713, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94f3ccf9f8a638fa974e1f69627832ef, server=bd6592c37bb2,40087,1732486423713 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T22:15:25,958 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40087 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94f3ccf9f8a638fa974e1f69627832ef, server=bd6592c37bb2,40087,1732486423713 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-24T22:15:25,958 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732486521610.94f3ccf9f8a638fa974e1f69627832ef., hostname=bd6592c37bb2,40087,1732486423713, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94f3ccf9f8a638fa974e1f69627832ef, server=bd6592c37bb2,40087,1732486423713 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T22:15:25,958 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732486521610.94f3ccf9f8a638fa974e1f69627832ef., hostname=bd6592c37bb2,40087,1732486423713, seqNum=5 because the exception is null or not the one we care about 2024-11-24T22:15:25,958 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40087 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:32822 deadline: 1732486585950, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94f3ccf9f8a638fa974e1f69627832ef, server=bd6592c37bb2,40087,1732486423713 2024-11-24T22:15:25,958 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732486521610.94f3ccf9f8a638fa974e1f69627832ef., hostname=bd6592c37bb2,40087,1732486423713, seqNum=5 , the old value is region=TestAcidGuarantees,,1732486521610.94f3ccf9f8a638fa974e1f69627832ef., hostname=bd6592c37bb2,40087,1732486423713, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94f3ccf9f8a638fa974e1f69627832ef, server=bd6592c37bb2,40087,1732486423713 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T22:15:25,958 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732486521610.94f3ccf9f8a638fa974e1f69627832ef., hostname=bd6592c37bb2,40087,1732486423713, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94f3ccf9f8a638fa974e1f69627832ef, server=bd6592c37bb2,40087,1732486423713 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T22:15:25,958 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732486521610.94f3ccf9f8a638fa974e1f69627832ef., hostname=bd6592c37bb2,40087,1732486423713, seqNum=5 because the exception is null or not the one we care about 2024-11-24T22:15:25,963 WARN [HBase-Metrics2-1 {}] impl.MetricsConfig(138): Cannot locate configuration: tried hadoop-metrics2-hbase.properties,hadoop-metrics2.properties 2024-11-24T22:15:25,971 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40087 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94f3ccf9f8a638fa974e1f69627832ef, server=bd6592c37bb2,40087,1732486423713 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-24T22:15:25,971 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40087 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:32846 deadline: 1732486585961, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94f3ccf9f8a638fa974e1f69627832ef, server=bd6592c37bb2,40087,1732486423713 2024-11-24T22:15:25,971 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40087 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94f3ccf9f8a638fa974e1f69627832ef, server=bd6592c37bb2,40087,1732486423713 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-24T22:15:25,971 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732486521610.94f3ccf9f8a638fa974e1f69627832ef., hostname=bd6592c37bb2,40087,1732486423713, seqNum=5 , the old value is region=TestAcidGuarantees,,1732486521610.94f3ccf9f8a638fa974e1f69627832ef., hostname=bd6592c37bb2,40087,1732486423713, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94f3ccf9f8a638fa974e1f69627832ef, server=bd6592c37bb2,40087,1732486423713 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T22:15:25,971 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40087 {}] ipc.CallRunner(138): callId: 11 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:32810 deadline: 1732486585961, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94f3ccf9f8a638fa974e1f69627832ef, server=bd6592c37bb2,40087,1732486423713 2024-11-24T22:15:25,971 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732486521610.94f3ccf9f8a638fa974e1f69627832ef., hostname=bd6592c37bb2,40087,1732486423713, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94f3ccf9f8a638fa974e1f69627832ef, server=bd6592c37bb2,40087,1732486423713 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T22:15:25,971 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732486521610.94f3ccf9f8a638fa974e1f69627832ef., hostname=bd6592c37bb2,40087,1732486423713, seqNum=5 because the exception is null or not the one we care about 2024-11-24T22:15:25,971 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732486521610.94f3ccf9f8a638fa974e1f69627832ef., hostname=bd6592c37bb2,40087,1732486423713, seqNum=5 , the old value is region=TestAcidGuarantees,,1732486521610.94f3ccf9f8a638fa974e1f69627832ef., hostname=bd6592c37bb2,40087,1732486423713, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94f3ccf9f8a638fa974e1f69627832ef, server=bd6592c37bb2,40087,1732486423713 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T22:15:25,972 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732486521610.94f3ccf9f8a638fa974e1f69627832ef., hostname=bd6592c37bb2,40087,1732486423713, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94f3ccf9f8a638fa974e1f69627832ef, server=bd6592c37bb2,40087,1732486423713 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T22:15:25,972 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732486521610.94f3ccf9f8a638fa974e1f69627832ef., hostname=bd6592c37bb2,40087,1732486423713, seqNum=5 because the exception is null or not the one we care about 2024-11-24T22:15:25,981 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40087 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94f3ccf9f8a638fa974e1f69627832ef, server=bd6592c37bb2,40087,1732486423713 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-24T22:15:25,982 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40087 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:32832 deadline: 1732486585971, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94f3ccf9f8a638fa974e1f69627832ef, server=bd6592c37bb2,40087,1732486423713 2024-11-24T22:15:25,982 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732486521610.94f3ccf9f8a638fa974e1f69627832ef., hostname=bd6592c37bb2,40087,1732486423713, seqNum=5 , the old value is region=TestAcidGuarantees,,1732486521610.94f3ccf9f8a638fa974e1f69627832ef., hostname=bd6592c37bb2,40087,1732486423713, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94f3ccf9f8a638fa974e1f69627832ef, server=bd6592c37bb2,40087,1732486423713 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T22:15:25,982 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732486521610.94f3ccf9f8a638fa974e1f69627832ef., hostname=bd6592c37bb2,40087,1732486423713, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94f3ccf9f8a638fa974e1f69627832ef, server=bd6592c37bb2,40087,1732486423713 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T22:15:25,982 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732486521610.94f3ccf9f8a638fa974e1f69627832ef., hostname=bd6592c37bb2,40087,1732486423713, seqNum=5 because the exception is null or not the one we care about 2024-11-24T22:15:26,016 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=16 (bloomFilter=true), to=hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/94f3ccf9f8a638fa974e1f69627832ef/.tmp/B/cd94decc27924b9687ddf2ec811ae351 2024-11-24T22:15:26,043 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/94f3ccf9f8a638fa974e1f69627832ef/.tmp/C/0c6077b470f14406ad8905d5895c98ee is 50, key is test_row_0/C:col10/1732486524716/Put/seqid=0 2024-11-24T22:15:26,046 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45929 is added to blk_1073742180_1356 (size=12001) 2024-11-24T22:15:26,088 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=40087 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=98 2024-11-24T22:15:26,088 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bd6592c37bb2:0-0 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732486521610.94f3ccf9f8a638fa974e1f69627832ef. 2024-11-24T22:15:26,088 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bd6592c37bb2:0-0 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732486521610.94f3ccf9f8a638fa974e1f69627832ef. as already flushing 2024-11-24T22:15:26,088 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bd6592c37bb2:0-0 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732486521610.94f3ccf9f8a638fa974e1f69627832ef. 2024-11-24T22:15:26,088 ERROR [RS_FLUSH_OPERATIONS-regionserver/bd6592c37bb2:0-0 {event_type=RS_FLUSH_REGIONS, pid=98}] handler.RSProcedureHandler(58): pid=98 java.io.IOException: Unable to complete flush {ENCODED => 94f3ccf9f8a638fa974e1f69627832ef, NAME => 'TestAcidGuarantees,,1732486521610.94f3ccf9f8a638fa974e1f69627832ef.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T22:15:26,088 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bd6592c37bb2:0-0 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=98 java.io.IOException: Unable to complete flush {ENCODED => 94f3ccf9f8a638fa974e1f69627832ef, NAME => 'TestAcidGuarantees,,1732486521610.94f3ccf9f8a638fa974e1f69627832ef.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T22:15:26,089 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37701 {}] master.HMaster(4177): Remote procedure failed, pid=98 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 94f3ccf9f8a638fa974e1f69627832ef, NAME => 'TestAcidGuarantees,,1732486521610.94f3ccf9f8a638fa974e1f69627832ef.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 94f3ccf9f8a638fa974e1f69627832ef, NAME => 'TestAcidGuarantees,,1732486521610.94f3ccf9f8a638fa974e1f69627832ef.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T22:15:26,241 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=40087 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=98 2024-11-24T22:15:26,241 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bd6592c37bb2:0-1 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732486521610.94f3ccf9f8a638fa974e1f69627832ef. 2024-11-24T22:15:26,242 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bd6592c37bb2:0-1 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732486521610.94f3ccf9f8a638fa974e1f69627832ef. as already flushing 2024-11-24T22:15:26,242 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bd6592c37bb2:0-1 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732486521610.94f3ccf9f8a638fa974e1f69627832ef. 2024-11-24T22:15:26,242 ERROR [RS_FLUSH_OPERATIONS-regionserver/bd6592c37bb2:0-1 {event_type=RS_FLUSH_REGIONS, pid=98}] handler.RSProcedureHandler(58): pid=98 java.io.IOException: Unable to complete flush {ENCODED => 94f3ccf9f8a638fa974e1f69627832ef, NAME => 'TestAcidGuarantees,,1732486521610.94f3ccf9f8a638fa974e1f69627832ef.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T22:15:26,242 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bd6592c37bb2:0-1 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=98 java.io.IOException: Unable to complete flush {ENCODED => 94f3ccf9f8a638fa974e1f69627832ef, NAME => 'TestAcidGuarantees,,1732486521610.94f3ccf9f8a638fa974e1f69627832ef.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T22:15:26,242 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37701 {}] master.HMaster(4177): Remote procedure failed, pid=98 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 94f3ccf9f8a638fa974e1f69627832ef, NAME => 'TestAcidGuarantees,,1732486521610.94f3ccf9f8a638fa974e1f69627832ef.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 94f3ccf9f8a638fa974e1f69627832ef, NAME => 'TestAcidGuarantees,,1732486521610.94f3ccf9f8a638fa974e1f69627832ef.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T22:15:26,394 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=40087 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=98 2024-11-24T22:15:26,394 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bd6592c37bb2:0-2 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732486521610.94f3ccf9f8a638fa974e1f69627832ef. 2024-11-24T22:15:26,394 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bd6592c37bb2:0-2 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732486521610.94f3ccf9f8a638fa974e1f69627832ef. as already flushing 2024-11-24T22:15:26,395 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bd6592c37bb2:0-2 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732486521610.94f3ccf9f8a638fa974e1f69627832ef. 2024-11-24T22:15:26,395 ERROR [RS_FLUSH_OPERATIONS-regionserver/bd6592c37bb2:0-2 {event_type=RS_FLUSH_REGIONS, pid=98}] handler.RSProcedureHandler(58): pid=98 java.io.IOException: Unable to complete flush {ENCODED => 94f3ccf9f8a638fa974e1f69627832ef, NAME => 'TestAcidGuarantees,,1732486521610.94f3ccf9f8a638fa974e1f69627832ef.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T22:15:26,395 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bd6592c37bb2:0-2 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=98 java.io.IOException: Unable to complete flush {ENCODED => 94f3ccf9f8a638fa974e1f69627832ef, NAME => 'TestAcidGuarantees,,1732486521610.94f3ccf9f8a638fa974e1f69627832ef.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T22:15:26,395 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37701 {}] master.HMaster(4177): Remote procedure failed, pid=98 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 94f3ccf9f8a638fa974e1f69627832ef, NAME => 'TestAcidGuarantees,,1732486521610.94f3ccf9f8a638fa974e1f69627832ef.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 94f3ccf9f8a638fa974e1f69627832ef, NAME => 'TestAcidGuarantees,,1732486521610.94f3ccf9f8a638fa974e1f69627832ef.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T22:15:26,447 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=16 (bloomFilter=true), to=hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/94f3ccf9f8a638fa974e1f69627832ef/.tmp/C/0c6077b470f14406ad8905d5895c98ee 2024-11-24T22:15:26,453 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/94f3ccf9f8a638fa974e1f69627832ef/.tmp/A/a8661fe5f09945f8996df4fc0affd5f9 as hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/94f3ccf9f8a638fa974e1f69627832ef/A/a8661fe5f09945f8996df4fc0affd5f9 2024-11-24T22:15:26,456 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/94f3ccf9f8a638fa974e1f69627832ef/A/a8661fe5f09945f8996df4fc0affd5f9, entries=150, sequenceid=16, filesize=30.2 K 2024-11-24T22:15:26,457 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/94f3ccf9f8a638fa974e1f69627832ef/.tmp/B/cd94decc27924b9687ddf2ec811ae351 as hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/94f3ccf9f8a638fa974e1f69627832ef/B/cd94decc27924b9687ddf2ec811ae351 2024-11-24T22:15:26,460 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/94f3ccf9f8a638fa974e1f69627832ef/B/cd94decc27924b9687ddf2ec811ae351, entries=150, sequenceid=16, filesize=11.7 K 2024-11-24T22:15:26,460 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/94f3ccf9f8a638fa974e1f69627832ef/.tmp/C/0c6077b470f14406ad8905d5895c98ee as hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/94f3ccf9f8a638fa974e1f69627832ef/C/0c6077b470f14406ad8905d5895c98ee 2024-11-24T22:15:26,464 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/94f3ccf9f8a638fa974e1f69627832ef/C/0c6077b470f14406ad8905d5895c98ee, entries=150, sequenceid=16, filesize=11.7 K 2024-11-24T22:15:26,465 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~60.38 KB/61830, heapSize ~180 KB/184320, currentSize=114.05 KB/116790 for 94f3ccf9f8a638fa974e1f69627832ef in 1735ms, sequenceid=16, compaction requested=false 2024-11-24T22:15:26,465 DEBUG [MemStoreFlusher.0 {}] regionserver.MetricsTableSourceImpl(133): Creating new MetricsTableSourceImpl for table 'TestAcidGuarantees' 2024-11-24T22:15:26,466 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 94f3ccf9f8a638fa974e1f69627832ef: 2024-11-24T22:15:26,546 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=40087 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=98 2024-11-24T22:15:26,546 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bd6592c37bb2:0-0 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732486521610.94f3ccf9f8a638fa974e1f69627832ef. 2024-11-24T22:15:26,547 INFO [RS_FLUSH_OPERATIONS-regionserver/bd6592c37bb2:0-0 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.HRegion(2902): Flushing 94f3ccf9f8a638fa974e1f69627832ef 3/3 column families, dataSize=114.05 KB heapSize=339.42 KB 2024-11-24T22:15:26,547 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bd6592c37bb2:0-0 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 94f3ccf9f8a638fa974e1f69627832ef, store=A 2024-11-24T22:15:26,547 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bd6592c37bb2:0-0 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-24T22:15:26,547 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bd6592c37bb2:0-0 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 94f3ccf9f8a638fa974e1f69627832ef, store=B 2024-11-24T22:15:26,547 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bd6592c37bb2:0-0 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-24T22:15:26,547 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bd6592c37bb2:0-0 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 94f3ccf9f8a638fa974e1f69627832ef, store=C 2024-11-24T22:15:26,547 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bd6592c37bb2:0-0 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-24T22:15:26,556 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bd6592c37bb2:0-0 {event_type=RS_FLUSH_REGIONS, pid=98}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411242bd23b0eed0e4f898ce86e11ba5d59a0_94f3ccf9f8a638fa974e1f69627832ef is 50, key is test_row_0/A:col10/1732486524803/Put/seqid=0 2024-11-24T22:15:26,562 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45929 is added to blk_1073742181_1357 (size=12154) 2024-11-24T22:15:26,851 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37701 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=97 2024-11-24T22:15:26,962 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bd6592c37bb2:0-0 {event_type=RS_FLUSH_REGIONS, pid=98}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T22:15:26,966 INFO [RS_FLUSH_OPERATIONS-regionserver/bd6592c37bb2:0-0 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411242bd23b0eed0e4f898ce86e11ba5d59a0_94f3ccf9f8a638fa974e1f69627832ef to hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411242bd23b0eed0e4f898ce86e11ba5d59a0_94f3ccf9f8a638fa974e1f69627832ef 2024-11-24T22:15:26,967 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bd6592c37bb2:0-0 {event_type=RS_FLUSH_REGIONS, pid=98}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/94f3ccf9f8a638fa974e1f69627832ef/.tmp/A/eea8340353634d808eee2f348943af0e, store: [table=TestAcidGuarantees family=A region=94f3ccf9f8a638fa974e1f69627832ef] 2024-11-24T22:15:26,968 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bd6592c37bb2:0-0 {event_type=RS_FLUSH_REGIONS, pid=98}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/94f3ccf9f8a638fa974e1f69627832ef/.tmp/A/eea8340353634d808eee2f348943af0e is 175, key is test_row_0/A:col10/1732486524803/Put/seqid=0 2024-11-24T22:15:26,973 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45929 is added to blk_1073742182_1358 (size=30955) 2024-11-24T22:15:26,975 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40087 {}] regionserver.HRegion(8855): Flush requested on 94f3ccf9f8a638fa974e1f69627832ef 2024-11-24T22:15:26,976 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732486521610.94f3ccf9f8a638fa974e1f69627832ef. as already flushing 2024-11-24T22:15:27,001 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40087 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94f3ccf9f8a638fa974e1f69627832ef, server=bd6592c37bb2,40087,1732486423713 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-24T22:15:27,001 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40087 {}] ipc.CallRunner(138): callId: 14 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:32822 deadline: 1732486586992, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94f3ccf9f8a638fa974e1f69627832ef, server=bd6592c37bb2,40087,1732486423713 2024-11-24T22:15:27,002 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732486521610.94f3ccf9f8a638fa974e1f69627832ef., hostname=bd6592c37bb2,40087,1732486423713, seqNum=5 , the old value is region=TestAcidGuarantees,,1732486521610.94f3ccf9f8a638fa974e1f69627832ef., hostname=bd6592c37bb2,40087,1732486423713, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94f3ccf9f8a638fa974e1f69627832ef, server=bd6592c37bb2,40087,1732486423713 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T22:15:27,002 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732486521610.94f3ccf9f8a638fa974e1f69627832ef., hostname=bd6592c37bb2,40087,1732486423713, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94f3ccf9f8a638fa974e1f69627832ef, server=bd6592c37bb2,40087,1732486423713 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T22:15:27,002 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732486521610.94f3ccf9f8a638fa974e1f69627832ef., hostname=bd6592c37bb2,40087,1732486423713, seqNum=5 because the exception is null or not the one we care about 2024-11-24T22:15:27,003 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40087 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94f3ccf9f8a638fa974e1f69627832ef, server=bd6592c37bb2,40087,1732486423713 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-24T22:15:27,003 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40087 {}] ipc.CallRunner(138): callId: 12 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:32846 deadline: 1732486586993, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94f3ccf9f8a638fa974e1f69627832ef, server=bd6592c37bb2,40087,1732486423713 2024-11-24T22:15:27,003 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40087 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94f3ccf9f8a638fa974e1f69627832ef, server=bd6592c37bb2,40087,1732486423713 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-24T22:15:27,003 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732486521610.94f3ccf9f8a638fa974e1f69627832ef., hostname=bd6592c37bb2,40087,1732486423713, seqNum=5 , the old value is region=TestAcidGuarantees,,1732486521610.94f3ccf9f8a638fa974e1f69627832ef., hostname=bd6592c37bb2,40087,1732486423713, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94f3ccf9f8a638fa974e1f69627832ef, server=bd6592c37bb2,40087,1732486423713 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T22:15:27,003 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40087 {}] ipc.CallRunner(138): callId: 15 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:32774 deadline: 1732486587000, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94f3ccf9f8a638fa974e1f69627832ef, server=bd6592c37bb2,40087,1732486423713 2024-11-24T22:15:27,003 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732486521610.94f3ccf9f8a638fa974e1f69627832ef., hostname=bd6592c37bb2,40087,1732486423713, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94f3ccf9f8a638fa974e1f69627832ef, server=bd6592c37bb2,40087,1732486423713 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T22:15:27,003 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732486521610.94f3ccf9f8a638fa974e1f69627832ef., hostname=bd6592c37bb2,40087,1732486423713, seqNum=5 because the exception is null or not the one we care about 2024-11-24T22:15:27,003 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732486521610.94f3ccf9f8a638fa974e1f69627832ef., hostname=bd6592c37bb2,40087,1732486423713, seqNum=5 , the old value is region=TestAcidGuarantees,,1732486521610.94f3ccf9f8a638fa974e1f69627832ef., hostname=bd6592c37bb2,40087,1732486423713, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94f3ccf9f8a638fa974e1f69627832ef, server=bd6592c37bb2,40087,1732486423713 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T22:15:27,003 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732486521610.94f3ccf9f8a638fa974e1f69627832ef., hostname=bd6592c37bb2,40087,1732486423713, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94f3ccf9f8a638fa974e1f69627832ef, server=bd6592c37bb2,40087,1732486423713 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T22:15:27,004 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732486521610.94f3ccf9f8a638fa974e1f69627832ef., hostname=bd6592c37bb2,40087,1732486423713, seqNum=5 because the exception is null or not the one we care about 2024-11-24T22:15:27,013 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40087 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94f3ccf9f8a638fa974e1f69627832ef, server=bd6592c37bb2,40087,1732486423713 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-24T22:15:27,013 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40087 {}] ipc.CallRunner(138): callId: 11 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:32832 deadline: 1732486587001, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94f3ccf9f8a638fa974e1f69627832ef, server=bd6592c37bb2,40087,1732486423713 2024-11-24T22:15:27,013 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40087 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94f3ccf9f8a638fa974e1f69627832ef, server=bd6592c37bb2,40087,1732486423713 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-24T22:15:27,014 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40087 {}] ipc.CallRunner(138): callId: 13 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:32810 deadline: 1732486587001, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94f3ccf9f8a638fa974e1f69627832ef, server=bd6592c37bb2,40087,1732486423713 2024-11-24T22:15:27,014 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732486521610.94f3ccf9f8a638fa974e1f69627832ef., hostname=bd6592c37bb2,40087,1732486423713, seqNum=5 , the old value is region=TestAcidGuarantees,,1732486521610.94f3ccf9f8a638fa974e1f69627832ef., hostname=bd6592c37bb2,40087,1732486423713, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94f3ccf9f8a638fa974e1f69627832ef, server=bd6592c37bb2,40087,1732486423713 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T22:15:27,014 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732486521610.94f3ccf9f8a638fa974e1f69627832ef., hostname=bd6592c37bb2,40087,1732486423713, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94f3ccf9f8a638fa974e1f69627832ef, server=bd6592c37bb2,40087,1732486423713 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T22:15:27,014 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732486521610.94f3ccf9f8a638fa974e1f69627832ef., hostname=bd6592c37bb2,40087,1732486423713, seqNum=5 because the exception is null or not the one we care about 2024-11-24T22:15:27,014 WARN [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRpcRetryingCaller(177): Call to bd6592c37bb2,40087,1732486423713 for 'test_row_1' in 94f3ccf9f8a638fa974e1f69627832ef of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2212 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94f3ccf9f8a638fa974e1f69627832ef, server=bd6592c37bb2,40087,1732486423713 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor44.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94f3ccf9f8a638fa974e1f69627832ef, server=bd6592c37bb2,40087,1732486423713 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-24T22:15:27,014 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732486521610.94f3ccf9f8a638fa974e1f69627832ef., hostname=bd6592c37bb2,40087,1732486423713, seqNum=5 , the old value is region=TestAcidGuarantees,,1732486521610.94f3ccf9f8a638fa974e1f69627832ef., hostname=bd6592c37bb2,40087,1732486423713, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94f3ccf9f8a638fa974e1f69627832ef, server=bd6592c37bb2,40087,1732486423713 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T22:15:27,014 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732486521610.94f3ccf9f8a638fa974e1f69627832ef., hostname=bd6592c37bb2,40087,1732486423713, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94f3ccf9f8a638fa974e1f69627832ef, server=bd6592c37bb2,40087,1732486423713 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T22:15:27,014 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732486521610.94f3ccf9f8a638fa974e1f69627832ef., hostname=bd6592c37bb2,40087,1732486423713, seqNum=5 because the exception is null or not the one we care about 2024-11-24T22:15:27,111 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40087 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94f3ccf9f8a638fa974e1f69627832ef, server=bd6592c37bb2,40087,1732486423713 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-24T22:15:27,112 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40087 {}] ipc.CallRunner(138): callId: 15 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:32822 deadline: 1732486587111, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94f3ccf9f8a638fa974e1f69627832ef, server=bd6592c37bb2,40087,1732486423713 2024-11-24T22:15:27,112 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40087 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94f3ccf9f8a638fa974e1f69627832ef, server=bd6592c37bb2,40087,1732486423713 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-24T22:15:27,112 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732486521610.94f3ccf9f8a638fa974e1f69627832ef., hostname=bd6592c37bb2,40087,1732486423713, seqNum=5 , the old value is region=TestAcidGuarantees,,1732486521610.94f3ccf9f8a638fa974e1f69627832ef., hostname=bd6592c37bb2,40087,1732486423713, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94f3ccf9f8a638fa974e1f69627832ef, server=bd6592c37bb2,40087,1732486423713 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T22:15:27,112 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40087 {}] ipc.CallRunner(138): callId: 13 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:32846 deadline: 1732486587111, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94f3ccf9f8a638fa974e1f69627832ef, server=bd6592c37bb2,40087,1732486423713 2024-11-24T22:15:27,112 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732486521610.94f3ccf9f8a638fa974e1f69627832ef., hostname=bd6592c37bb2,40087,1732486423713, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94f3ccf9f8a638fa974e1f69627832ef, server=bd6592c37bb2,40087,1732486423713 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T22:15:27,112 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732486521610.94f3ccf9f8a638fa974e1f69627832ef., hostname=bd6592c37bb2,40087,1732486423713, seqNum=5 because the exception is null or not the one we care about 2024-11-24T22:15:27,112 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732486521610.94f3ccf9f8a638fa974e1f69627832ef., hostname=bd6592c37bb2,40087,1732486423713, seqNum=5 , the old value is region=TestAcidGuarantees,,1732486521610.94f3ccf9f8a638fa974e1f69627832ef., hostname=bd6592c37bb2,40087,1732486423713, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94f3ccf9f8a638fa974e1f69627832ef, server=bd6592c37bb2,40087,1732486423713 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T22:15:27,112 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732486521610.94f3ccf9f8a638fa974e1f69627832ef., hostname=bd6592c37bb2,40087,1732486423713, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94f3ccf9f8a638fa974e1f69627832ef, server=bd6592c37bb2,40087,1732486423713 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T22:15:27,112 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732486521610.94f3ccf9f8a638fa974e1f69627832ef., hostname=bd6592c37bb2,40087,1732486423713, seqNum=5 because the exception is null or not the one we care about 2024-11-24T22:15:27,120 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40087 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94f3ccf9f8a638fa974e1f69627832ef, server=bd6592c37bb2,40087,1732486423713 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-24T22:15:27,120 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40087 {}] ipc.CallRunner(138): callId: 16 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:32774 deadline: 1732486587111, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94f3ccf9f8a638fa974e1f69627832ef, server=bd6592c37bb2,40087,1732486423713 2024-11-24T22:15:27,120 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732486521610.94f3ccf9f8a638fa974e1f69627832ef., hostname=bd6592c37bb2,40087,1732486423713, seqNum=5 , the old value is region=TestAcidGuarantees,,1732486521610.94f3ccf9f8a638fa974e1f69627832ef., hostname=bd6592c37bb2,40087,1732486423713, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94f3ccf9f8a638fa974e1f69627832ef, server=bd6592c37bb2,40087,1732486423713 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T22:15:27,121 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732486521610.94f3ccf9f8a638fa974e1f69627832ef., hostname=bd6592c37bb2,40087,1732486423713, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94f3ccf9f8a638fa974e1f69627832ef, server=bd6592c37bb2,40087,1732486423713 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T22:15:27,121 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732486521610.94f3ccf9f8a638fa974e1f69627832ef., hostname=bd6592c37bb2,40087,1732486423713, seqNum=5 because the exception is null or not the one we care about 2024-11-24T22:15:27,124 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40087 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94f3ccf9f8a638fa974e1f69627832ef, server=bd6592c37bb2,40087,1732486423713 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-24T22:15:27,124 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40087 {}] ipc.CallRunner(138): callId: 14 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:32810 deadline: 1732486587120, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94f3ccf9f8a638fa974e1f69627832ef, server=bd6592c37bb2,40087,1732486423713 2024-11-24T22:15:27,124 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732486521610.94f3ccf9f8a638fa974e1f69627832ef., hostname=bd6592c37bb2,40087,1732486423713, seqNum=5 , the old value is region=TestAcidGuarantees,,1732486521610.94f3ccf9f8a638fa974e1f69627832ef., hostname=bd6592c37bb2,40087,1732486423713, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94f3ccf9f8a638fa974e1f69627832ef, server=bd6592c37bb2,40087,1732486423713 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T22:15:27,124 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732486521610.94f3ccf9f8a638fa974e1f69627832ef., hostname=bd6592c37bb2,40087,1732486423713, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94f3ccf9f8a638fa974e1f69627832ef, server=bd6592c37bb2,40087,1732486423713 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T22:15:27,124 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732486521610.94f3ccf9f8a638fa974e1f69627832ef., hostname=bd6592c37bb2,40087,1732486423713, seqNum=5 because the exception is null or not the one we care about 2024-11-24T22:15:27,329 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40087 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94f3ccf9f8a638fa974e1f69627832ef, server=bd6592c37bb2,40087,1732486423713 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-24T22:15:27,329 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40087 {}] ipc.CallRunner(138): callId: 16 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:32822 deadline: 1732486587321, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94f3ccf9f8a638fa974e1f69627832ef, server=bd6592c37bb2,40087,1732486423713 2024-11-24T22:15:27,329 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40087 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94f3ccf9f8a638fa974e1f69627832ef, server=bd6592c37bb2,40087,1732486423713 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-24T22:15:27,329 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732486521610.94f3ccf9f8a638fa974e1f69627832ef., hostname=bd6592c37bb2,40087,1732486423713, seqNum=5 , the old value is region=TestAcidGuarantees,,1732486521610.94f3ccf9f8a638fa974e1f69627832ef., hostname=bd6592c37bb2,40087,1732486423713, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94f3ccf9f8a638fa974e1f69627832ef, server=bd6592c37bb2,40087,1732486423713 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T22:15:27,329 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40087 {}] ipc.CallRunner(138): callId: 14 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:32846 deadline: 1732486587321, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94f3ccf9f8a638fa974e1f69627832ef, server=bd6592c37bb2,40087,1732486423713 2024-11-24T22:15:27,329 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732486521610.94f3ccf9f8a638fa974e1f69627832ef., hostname=bd6592c37bb2,40087,1732486423713, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94f3ccf9f8a638fa974e1f69627832ef, server=bd6592c37bb2,40087,1732486423713 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T22:15:27,329 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732486521610.94f3ccf9f8a638fa974e1f69627832ef., hostname=bd6592c37bb2,40087,1732486423713, seqNum=5 because the exception is null or not the one we care about 2024-11-24T22:15:27,329 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732486521610.94f3ccf9f8a638fa974e1f69627832ef., hostname=bd6592c37bb2,40087,1732486423713, seqNum=5 , the old value is region=TestAcidGuarantees,,1732486521610.94f3ccf9f8a638fa974e1f69627832ef., hostname=bd6592c37bb2,40087,1732486423713, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94f3ccf9f8a638fa974e1f69627832ef, server=bd6592c37bb2,40087,1732486423713 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T22:15:27,329 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732486521610.94f3ccf9f8a638fa974e1f69627832ef., hostname=bd6592c37bb2,40087,1732486423713, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94f3ccf9f8a638fa974e1f69627832ef, server=bd6592c37bb2,40087,1732486423713 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T22:15:27,329 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732486521610.94f3ccf9f8a638fa974e1f69627832ef., hostname=bd6592c37bb2,40087,1732486423713, seqNum=5 because the exception is null or not the one we care about 2024-11-24T22:15:27,338 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40087 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94f3ccf9f8a638fa974e1f69627832ef, server=bd6592c37bb2,40087,1732486423713 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-24T22:15:27,338 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40087 {}] ipc.CallRunner(138): callId: 17 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:32774 deadline: 1732486587330, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94f3ccf9f8a638fa974e1f69627832ef, server=bd6592c37bb2,40087,1732486423713 2024-11-24T22:15:27,338 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732486521610.94f3ccf9f8a638fa974e1f69627832ef., hostname=bd6592c37bb2,40087,1732486423713, seqNum=5 , the old value is region=TestAcidGuarantees,,1732486521610.94f3ccf9f8a638fa974e1f69627832ef., hostname=bd6592c37bb2,40087,1732486423713, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94f3ccf9f8a638fa974e1f69627832ef, server=bd6592c37bb2,40087,1732486423713 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T22:15:27,338 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40087 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94f3ccf9f8a638fa974e1f69627832ef, server=bd6592c37bb2,40087,1732486423713 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-24T22:15:27,338 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732486521610.94f3ccf9f8a638fa974e1f69627832ef., hostname=bd6592c37bb2,40087,1732486423713, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94f3ccf9f8a638fa974e1f69627832ef, server=bd6592c37bb2,40087,1732486423713 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T22:15:27,338 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732486521610.94f3ccf9f8a638fa974e1f69627832ef., hostname=bd6592c37bb2,40087,1732486423713, seqNum=5 because the exception is null or not the one we care about 2024-11-24T22:15:27,338 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40087 {}] ipc.CallRunner(138): callId: 15 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:32810 deadline: 1732486587330, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94f3ccf9f8a638fa974e1f69627832ef, server=bd6592c37bb2,40087,1732486423713 2024-11-24T22:15:27,339 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732486521610.94f3ccf9f8a638fa974e1f69627832ef., hostname=bd6592c37bb2,40087,1732486423713, seqNum=5 , the old value is region=TestAcidGuarantees,,1732486521610.94f3ccf9f8a638fa974e1f69627832ef., hostname=bd6592c37bb2,40087,1732486423713, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94f3ccf9f8a638fa974e1f69627832ef, server=bd6592c37bb2,40087,1732486423713 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T22:15:27,339 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732486521610.94f3ccf9f8a638fa974e1f69627832ef., hostname=bd6592c37bb2,40087,1732486423713, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94f3ccf9f8a638fa974e1f69627832ef, server=bd6592c37bb2,40087,1732486423713 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T22:15:27,339 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732486521610.94f3ccf9f8a638fa974e1f69627832ef., hostname=bd6592c37bb2,40087,1732486423713, seqNum=5 because the exception is null or not the one we care about 2024-11-24T22:15:27,373 INFO [RS_FLUSH_OPERATIONS-regionserver/bd6592c37bb2:0-0 {event_type=RS_FLUSH_REGIONS, pid=98}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=36, memsize=38.0 K, hasBloomFilter=true, into tmp file hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/94f3ccf9f8a638fa974e1f69627832ef/.tmp/A/eea8340353634d808eee2f348943af0e 2024-11-24T22:15:27,381 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bd6592c37bb2:0-0 {event_type=RS_FLUSH_REGIONS, pid=98}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/94f3ccf9f8a638fa974e1f69627832ef/.tmp/B/cc5afda838d84551b7b988a9ea3631f6 is 50, key is test_row_0/B:col10/1732486524803/Put/seqid=0 2024-11-24T22:15:27,385 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45929 is added to blk_1073742183_1359 (size=12001) 2024-11-24T22:15:27,649 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40087 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94f3ccf9f8a638fa974e1f69627832ef, server=bd6592c37bb2,40087,1732486423713 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-24T22:15:27,649 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40087 {}] ipc.CallRunner(138): callId: 17 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:32822 deadline: 1732486587641, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94f3ccf9f8a638fa974e1f69627832ef, server=bd6592c37bb2,40087,1732486423713 2024-11-24T22:15:27,650 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732486521610.94f3ccf9f8a638fa974e1f69627832ef., hostname=bd6592c37bb2,40087,1732486423713, seqNum=5 , the old value is region=TestAcidGuarantees,,1732486521610.94f3ccf9f8a638fa974e1f69627832ef., hostname=bd6592c37bb2,40087,1732486423713, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94f3ccf9f8a638fa974e1f69627832ef, server=bd6592c37bb2,40087,1732486423713 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T22:15:27,649 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40087 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94f3ccf9f8a638fa974e1f69627832ef, server=bd6592c37bb2,40087,1732486423713 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-24T22:15:27,650 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732486521610.94f3ccf9f8a638fa974e1f69627832ef., hostname=bd6592c37bb2,40087,1732486423713, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94f3ccf9f8a638fa974e1f69627832ef, server=bd6592c37bb2,40087,1732486423713 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T22:15:27,650 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732486521610.94f3ccf9f8a638fa974e1f69627832ef., hostname=bd6592c37bb2,40087,1732486423713, seqNum=5 because the exception is null or not the one we care about 2024-11-24T22:15:27,650 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40087 {}] ipc.CallRunner(138): callId: 15 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:32846 deadline: 1732486587641, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94f3ccf9f8a638fa974e1f69627832ef, server=bd6592c37bb2,40087,1732486423713 2024-11-24T22:15:27,650 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732486521610.94f3ccf9f8a638fa974e1f69627832ef., hostname=bd6592c37bb2,40087,1732486423713, seqNum=5 , the old value is region=TestAcidGuarantees,,1732486521610.94f3ccf9f8a638fa974e1f69627832ef., hostname=bd6592c37bb2,40087,1732486423713, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94f3ccf9f8a638fa974e1f69627832ef, server=bd6592c37bb2,40087,1732486423713 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T22:15:27,650 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732486521610.94f3ccf9f8a638fa974e1f69627832ef., hostname=bd6592c37bb2,40087,1732486423713, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94f3ccf9f8a638fa974e1f69627832ef, server=bd6592c37bb2,40087,1732486423713 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T22:15:27,650 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732486521610.94f3ccf9f8a638fa974e1f69627832ef., hostname=bd6592c37bb2,40087,1732486423713, seqNum=5 because the exception is null or not the one we care about 2024-11-24T22:15:27,650 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40087 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94f3ccf9f8a638fa974e1f69627832ef, server=bd6592c37bb2,40087,1732486423713 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-24T22:15:27,650 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40087 {}] ipc.CallRunner(138): callId: 18 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:32774 deadline: 1732486587641, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94f3ccf9f8a638fa974e1f69627832ef, server=bd6592c37bb2,40087,1732486423713 2024-11-24T22:15:27,650 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732486521610.94f3ccf9f8a638fa974e1f69627832ef., hostname=bd6592c37bb2,40087,1732486423713, seqNum=5 , the old value is region=TestAcidGuarantees,,1732486521610.94f3ccf9f8a638fa974e1f69627832ef., hostname=bd6592c37bb2,40087,1732486423713, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94f3ccf9f8a638fa974e1f69627832ef, server=bd6592c37bb2,40087,1732486423713 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T22:15:27,650 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732486521610.94f3ccf9f8a638fa974e1f69627832ef., hostname=bd6592c37bb2,40087,1732486423713, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94f3ccf9f8a638fa974e1f69627832ef, server=bd6592c37bb2,40087,1732486423713 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T22:15:27,650 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732486521610.94f3ccf9f8a638fa974e1f69627832ef., hostname=bd6592c37bb2,40087,1732486423713, seqNum=5 because the exception is null or not the one we care about 2024-11-24T22:15:27,650 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40087 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94f3ccf9f8a638fa974e1f69627832ef, server=bd6592c37bb2,40087,1732486423713 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-24T22:15:27,650 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40087 {}] ipc.CallRunner(138): callId: 16 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:32810 deadline: 1732486587641, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94f3ccf9f8a638fa974e1f69627832ef, server=bd6592c37bb2,40087,1732486423713 2024-11-24T22:15:27,651 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732486521610.94f3ccf9f8a638fa974e1f69627832ef., hostname=bd6592c37bb2,40087,1732486423713, seqNum=5 , the old value is region=TestAcidGuarantees,,1732486521610.94f3ccf9f8a638fa974e1f69627832ef., hostname=bd6592c37bb2,40087,1732486423713, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94f3ccf9f8a638fa974e1f69627832ef, server=bd6592c37bb2,40087,1732486423713 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T22:15:27,651 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732486521610.94f3ccf9f8a638fa974e1f69627832ef., hostname=bd6592c37bb2,40087,1732486423713, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94f3ccf9f8a638fa974e1f69627832ef, server=bd6592c37bb2,40087,1732486423713 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T22:15:27,651 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732486521610.94f3ccf9f8a638fa974e1f69627832ef., hostname=bd6592c37bb2,40087,1732486423713, seqNum=5 because the exception is null or not the one we care about 2024-11-24T22:15:27,786 INFO [RS_FLUSH_OPERATIONS-regionserver/bd6592c37bb2:0-0 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=38.02 KB at sequenceid=36 (bloomFilter=true), to=hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/94f3ccf9f8a638fa974e1f69627832ef/.tmp/B/cc5afda838d84551b7b988a9ea3631f6 2024-11-24T22:15:27,793 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bd6592c37bb2:0-0 {event_type=RS_FLUSH_REGIONS, pid=98}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/94f3ccf9f8a638fa974e1f69627832ef/.tmp/C/a6f6fe973984491a9e1d7cecd110be45 is 50, key is test_row_0/C:col10/1732486524803/Put/seqid=0 2024-11-24T22:15:27,796 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45929 is added to blk_1073742184_1360 (size=12001) 2024-11-24T22:15:27,797 INFO [RS_FLUSH_OPERATIONS-regionserver/bd6592c37bb2:0-0 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=38.02 KB at sequenceid=36 (bloomFilter=true), to=hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/94f3ccf9f8a638fa974e1f69627832ef/.tmp/C/a6f6fe973984491a9e1d7cecd110be45 2024-11-24T22:15:27,804 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bd6592c37bb2:0-0 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/94f3ccf9f8a638fa974e1f69627832ef/.tmp/A/eea8340353634d808eee2f348943af0e as hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/94f3ccf9f8a638fa974e1f69627832ef/A/eea8340353634d808eee2f348943af0e 2024-11-24T22:15:27,807 INFO [RS_FLUSH_OPERATIONS-regionserver/bd6592c37bb2:0-0 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/94f3ccf9f8a638fa974e1f69627832ef/A/eea8340353634d808eee2f348943af0e, entries=150, sequenceid=36, filesize=30.2 K 2024-11-24T22:15:27,808 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bd6592c37bb2:0-0 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/94f3ccf9f8a638fa974e1f69627832ef/.tmp/B/cc5afda838d84551b7b988a9ea3631f6 as hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/94f3ccf9f8a638fa974e1f69627832ef/B/cc5afda838d84551b7b988a9ea3631f6 2024-11-24T22:15:27,811 INFO [RS_FLUSH_OPERATIONS-regionserver/bd6592c37bb2:0-0 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/94f3ccf9f8a638fa974e1f69627832ef/B/cc5afda838d84551b7b988a9ea3631f6, entries=150, sequenceid=36, filesize=11.7 K 2024-11-24T22:15:27,812 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bd6592c37bb2:0-0 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/94f3ccf9f8a638fa974e1f69627832ef/.tmp/C/a6f6fe973984491a9e1d7cecd110be45 as hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/94f3ccf9f8a638fa974e1f69627832ef/C/a6f6fe973984491a9e1d7cecd110be45 2024-11-24T22:15:27,815 INFO [RS_FLUSH_OPERATIONS-regionserver/bd6592c37bb2:0-0 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/94f3ccf9f8a638fa974e1f69627832ef/C/a6f6fe973984491a9e1d7cecd110be45, entries=150, sequenceid=36, filesize=11.7 K 2024-11-24T22:15:27,816 INFO [RS_FLUSH_OPERATIONS-regionserver/bd6592c37bb2:0-0 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.HRegion(3140): Finished flush of dataSize ~114.05 KB/116790, heapSize ~339.38 KB/347520, currentSize=60.38 KB/61830 for 94f3ccf9f8a638fa974e1f69627832ef in 1269ms, sequenceid=36, compaction requested=false 2024-11-24T22:15:27,816 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bd6592c37bb2:0-0 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.HRegion(2603): Flush status journal for 94f3ccf9f8a638fa974e1f69627832ef: 2024-11-24T22:15:27,816 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bd6592c37bb2:0-0 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732486521610.94f3ccf9f8a638fa974e1f69627832ef. 2024-11-24T22:15:27,816 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bd6592c37bb2:0-0 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=98 2024-11-24T22:15:27,817 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37701 {}] master.HMaster(4169): Remote procedure done, pid=98 2024-11-24T22:15:27,819 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=98, resume processing ppid=97 2024-11-24T22:15:27,819 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=98, ppid=97, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 3.1090 sec 2024-11-24T22:15:27,821 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=97, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=97, table=TestAcidGuarantees in 3.1150 sec 2024-11-24T22:15:28,165 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40087 {}] regionserver.HRegion(8855): Flush requested on 94f3ccf9f8a638fa974e1f69627832ef 2024-11-24T22:15:28,165 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 94f3ccf9f8a638fa974e1f69627832ef 3/3 column families, dataSize=67.09 KB heapSize=199.97 KB 2024-11-24T22:15:28,165 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 94f3ccf9f8a638fa974e1f69627832ef, store=A 2024-11-24T22:15:28,165 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-24T22:15:28,165 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 94f3ccf9f8a638fa974e1f69627832ef, store=B 2024-11-24T22:15:28,165 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-24T22:15:28,165 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 94f3ccf9f8a638fa974e1f69627832ef, store=C 2024-11-24T22:15:28,165 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-24T22:15:28,172 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241124d09a5d75e35d448e8a2bc00040a143ab_94f3ccf9f8a638fa974e1f69627832ef is 50, key is test_row_0/A:col10/1732486526999/Put/seqid=0 2024-11-24T22:15:28,175 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45929 is added to blk_1073742185_1361 (size=14594) 2024-11-24T22:15:28,176 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T22:15:28,190 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241124d09a5d75e35d448e8a2bc00040a143ab_94f3ccf9f8a638fa974e1f69627832ef to hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241124d09a5d75e35d448e8a2bc00040a143ab_94f3ccf9f8a638fa974e1f69627832ef 2024-11-24T22:15:28,191 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/94f3ccf9f8a638fa974e1f69627832ef/.tmp/A/8ce2903e8a5b46edb21934de4a278e2d, store: [table=TestAcidGuarantees family=A region=94f3ccf9f8a638fa974e1f69627832ef] 2024-11-24T22:15:28,192 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/94f3ccf9f8a638fa974e1f69627832ef/.tmp/A/8ce2903e8a5b46edb21934de4a278e2d is 175, key is test_row_0/A:col10/1732486526999/Put/seqid=0 2024-11-24T22:15:28,195 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45929 is added to blk_1073742186_1362 (size=39549) 2024-11-24T22:15:28,196 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=49, memsize=22.4 K, hasBloomFilter=true, into tmp file hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/94f3ccf9f8a638fa974e1f69627832ef/.tmp/A/8ce2903e8a5b46edb21934de4a278e2d 2024-11-24T22:15:28,206 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/94f3ccf9f8a638fa974e1f69627832ef/.tmp/B/178dfc664e9c455d93e59f22305934cf is 50, key is test_row_0/B:col10/1732486526999/Put/seqid=0 2024-11-24T22:15:28,214 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45929 is added to blk_1073742187_1363 (size=12001) 2024-11-24T22:15:28,215 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=49 (bloomFilter=true), to=hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/94f3ccf9f8a638fa974e1f69627832ef/.tmp/B/178dfc664e9c455d93e59f22305934cf 2024-11-24T22:15:28,217 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40087 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94f3ccf9f8a638fa974e1f69627832ef, server=bd6592c37bb2,40087,1732486423713 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-24T22:15:28,217 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40087 {}] ipc.CallRunner(138): callId: 22 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:32774 deadline: 1732486588204, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94f3ccf9f8a638fa974e1f69627832ef, server=bd6592c37bb2,40087,1732486423713 2024-11-24T22:15:28,217 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732486521610.94f3ccf9f8a638fa974e1f69627832ef., hostname=bd6592c37bb2,40087,1732486423713, seqNum=5 , the old value is region=TestAcidGuarantees,,1732486521610.94f3ccf9f8a638fa974e1f69627832ef., hostname=bd6592c37bb2,40087,1732486423713, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94f3ccf9f8a638fa974e1f69627832ef, server=bd6592c37bb2,40087,1732486423713 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T22:15:28,217 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732486521610.94f3ccf9f8a638fa974e1f69627832ef., hostname=bd6592c37bb2,40087,1732486423713, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94f3ccf9f8a638fa974e1f69627832ef, server=bd6592c37bb2,40087,1732486423713 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T22:15:28,217 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732486521610.94f3ccf9f8a638fa974e1f69627832ef., hostname=bd6592c37bb2,40087,1732486423713, seqNum=5 because the exception is null or not the one we care about 2024-11-24T22:15:28,223 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40087 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94f3ccf9f8a638fa974e1f69627832ef, server=bd6592c37bb2,40087,1732486423713 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-24T22:15:28,223 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/94f3ccf9f8a638fa974e1f69627832ef/.tmp/C/2ba25fa3975e49de8bfb037efe930ba5 is 50, key is test_row_0/C:col10/1732486526999/Put/seqid=0 2024-11-24T22:15:28,224 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40087 {}] ipc.CallRunner(138): callId: 22 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:32822 deadline: 1732486588205, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94f3ccf9f8a638fa974e1f69627832ef, server=bd6592c37bb2,40087,1732486423713 2024-11-24T22:15:28,224 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732486521610.94f3ccf9f8a638fa974e1f69627832ef., hostname=bd6592c37bb2,40087,1732486423713, seqNum=5 , the old value is region=TestAcidGuarantees,,1732486521610.94f3ccf9f8a638fa974e1f69627832ef., hostname=bd6592c37bb2,40087,1732486423713, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94f3ccf9f8a638fa974e1f69627832ef, server=bd6592c37bb2,40087,1732486423713 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T22:15:28,224 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732486521610.94f3ccf9f8a638fa974e1f69627832ef., hostname=bd6592c37bb2,40087,1732486423713, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94f3ccf9f8a638fa974e1f69627832ef, server=bd6592c37bb2,40087,1732486423713 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T22:15:28,224 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732486521610.94f3ccf9f8a638fa974e1f69627832ef., hostname=bd6592c37bb2,40087,1732486423713, seqNum=5 because the exception is null or not the one we care about 2024-11-24T22:15:28,229 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40087 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94f3ccf9f8a638fa974e1f69627832ef, server=bd6592c37bb2,40087,1732486423713 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-24T22:15:28,229 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40087 {}] ipc.CallRunner(138): callId: 21 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:32846 deadline: 1732486588214, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94f3ccf9f8a638fa974e1f69627832ef, server=bd6592c37bb2,40087,1732486423713 2024-11-24T22:15:28,229 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40087 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94f3ccf9f8a638fa974e1f69627832ef, server=bd6592c37bb2,40087,1732486423713 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-24T22:15:28,229 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40087 {}] ipc.CallRunner(138): callId: 22 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:32810 deadline: 1732486588217, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94f3ccf9f8a638fa974e1f69627832ef, server=bd6592c37bb2,40087,1732486423713 2024-11-24T22:15:28,229 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732486521610.94f3ccf9f8a638fa974e1f69627832ef., hostname=bd6592c37bb2,40087,1732486423713, seqNum=5 , the old value is region=TestAcidGuarantees,,1732486521610.94f3ccf9f8a638fa974e1f69627832ef., hostname=bd6592c37bb2,40087,1732486423713, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94f3ccf9f8a638fa974e1f69627832ef, server=bd6592c37bb2,40087,1732486423713 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T22:15:28,229 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732486521610.94f3ccf9f8a638fa974e1f69627832ef., hostname=bd6592c37bb2,40087,1732486423713, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94f3ccf9f8a638fa974e1f69627832ef, server=bd6592c37bb2,40087,1732486423713 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T22:15:28,230 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732486521610.94f3ccf9f8a638fa974e1f69627832ef., hostname=bd6592c37bb2,40087,1732486423713, seqNum=5 because the exception is null or not the one we care about 2024-11-24T22:15:28,230 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732486521610.94f3ccf9f8a638fa974e1f69627832ef., hostname=bd6592c37bb2,40087,1732486423713, seqNum=5 , the old value is region=TestAcidGuarantees,,1732486521610.94f3ccf9f8a638fa974e1f69627832ef., hostname=bd6592c37bb2,40087,1732486423713, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94f3ccf9f8a638fa974e1f69627832ef, server=bd6592c37bb2,40087,1732486423713 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T22:15:28,230 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732486521610.94f3ccf9f8a638fa974e1f69627832ef., hostname=bd6592c37bb2,40087,1732486423713, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94f3ccf9f8a638fa974e1f69627832ef, server=bd6592c37bb2,40087,1732486423713 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T22:15:28,230 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732486521610.94f3ccf9f8a638fa974e1f69627832ef., hostname=bd6592c37bb2,40087,1732486423713, seqNum=5 because the exception is null or not the one we care about 2024-11-24T22:15:28,237 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45929 is added to blk_1073742188_1364 (size=12001) 2024-11-24T22:15:28,331 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40087 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94f3ccf9f8a638fa974e1f69627832ef, server=bd6592c37bb2,40087,1732486423713 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-24T22:15:28,331 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40087 {}] ipc.CallRunner(138): callId: 23 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:32774 deadline: 1732486588320, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94f3ccf9f8a638fa974e1f69627832ef, server=bd6592c37bb2,40087,1732486423713 2024-11-24T22:15:28,331 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732486521610.94f3ccf9f8a638fa974e1f69627832ef., hostname=bd6592c37bb2,40087,1732486423713, seqNum=5 , the old value is region=TestAcidGuarantees,,1732486521610.94f3ccf9f8a638fa974e1f69627832ef., hostname=bd6592c37bb2,40087,1732486423713, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94f3ccf9f8a638fa974e1f69627832ef, server=bd6592c37bb2,40087,1732486423713 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T22:15:28,331 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732486521610.94f3ccf9f8a638fa974e1f69627832ef., hostname=bd6592c37bb2,40087,1732486423713, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94f3ccf9f8a638fa974e1f69627832ef, server=bd6592c37bb2,40087,1732486423713 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T22:15:28,331 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732486521610.94f3ccf9f8a638fa974e1f69627832ef., hostname=bd6592c37bb2,40087,1732486423713, seqNum=5 because the exception is null or not the one we care about 2024-11-24T22:15:28,340 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40087 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94f3ccf9f8a638fa974e1f69627832ef, server=bd6592c37bb2,40087,1732486423713 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-24T22:15:28,340 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40087 {}] ipc.CallRunner(138): callId: 23 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:32822 deadline: 1732486588331, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94f3ccf9f8a638fa974e1f69627832ef, server=bd6592c37bb2,40087,1732486423713 2024-11-24T22:15:28,340 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732486521610.94f3ccf9f8a638fa974e1f69627832ef., hostname=bd6592c37bb2,40087,1732486423713, seqNum=5 , the old value is region=TestAcidGuarantees,,1732486521610.94f3ccf9f8a638fa974e1f69627832ef., hostname=bd6592c37bb2,40087,1732486423713, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94f3ccf9f8a638fa974e1f69627832ef, server=bd6592c37bb2,40087,1732486423713 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T22:15:28,341 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732486521610.94f3ccf9f8a638fa974e1f69627832ef., hostname=bd6592c37bb2,40087,1732486423713, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94f3ccf9f8a638fa974e1f69627832ef, server=bd6592c37bb2,40087,1732486423713 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T22:15:28,341 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732486521610.94f3ccf9f8a638fa974e1f69627832ef., hostname=bd6592c37bb2,40087,1732486423713, seqNum=5 because the exception is null or not the one we care about 2024-11-24T22:15:28,348 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40087 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94f3ccf9f8a638fa974e1f69627832ef, server=bd6592c37bb2,40087,1732486423713 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-24T22:15:28,348 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40087 {}] ipc.CallRunner(138): callId: 22 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:32846 deadline: 1732486588341, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94f3ccf9f8a638fa974e1f69627832ef, server=bd6592c37bb2,40087,1732486423713 2024-11-24T22:15:28,348 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732486521610.94f3ccf9f8a638fa974e1f69627832ef., hostname=bd6592c37bb2,40087,1732486423713, seqNum=5 , the old value is region=TestAcidGuarantees,,1732486521610.94f3ccf9f8a638fa974e1f69627832ef., hostname=bd6592c37bb2,40087,1732486423713, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94f3ccf9f8a638fa974e1f69627832ef, server=bd6592c37bb2,40087,1732486423713 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T22:15:28,348 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40087 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94f3ccf9f8a638fa974e1f69627832ef, server=bd6592c37bb2,40087,1732486423713 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-24T22:15:28,349 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732486521610.94f3ccf9f8a638fa974e1f69627832ef., hostname=bd6592c37bb2,40087,1732486423713, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94f3ccf9f8a638fa974e1f69627832ef, server=bd6592c37bb2,40087,1732486423713 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T22:15:28,349 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732486521610.94f3ccf9f8a638fa974e1f69627832ef., hostname=bd6592c37bb2,40087,1732486423713, seqNum=5 because the exception is null or not the one we care about 2024-11-24T22:15:28,349 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40087 {}] ipc.CallRunner(138): callId: 23 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:32810 deadline: 1732486588341, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94f3ccf9f8a638fa974e1f69627832ef, server=bd6592c37bb2,40087,1732486423713 2024-11-24T22:15:28,349 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732486521610.94f3ccf9f8a638fa974e1f69627832ef., hostname=bd6592c37bb2,40087,1732486423713, seqNum=5 , the old value is region=TestAcidGuarantees,,1732486521610.94f3ccf9f8a638fa974e1f69627832ef., hostname=bd6592c37bb2,40087,1732486423713, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94f3ccf9f8a638fa974e1f69627832ef, server=bd6592c37bb2,40087,1732486423713 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T22:15:28,349 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732486521610.94f3ccf9f8a638fa974e1f69627832ef., hostname=bd6592c37bb2,40087,1732486423713, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94f3ccf9f8a638fa974e1f69627832ef, server=bd6592c37bb2,40087,1732486423713 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T22:15:28,349 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732486521610.94f3ccf9f8a638fa974e1f69627832ef., hostname=bd6592c37bb2,40087,1732486423713, seqNum=5 because the exception is null or not the one we care about 2024-11-24T22:15:28,552 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40087 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94f3ccf9f8a638fa974e1f69627832ef, server=bd6592c37bb2,40087,1732486423713 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-24T22:15:28,552 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40087 {}] ipc.CallRunner(138): callId: 24 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:32774 deadline: 1732486588541, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94f3ccf9f8a638fa974e1f69627832ef, server=bd6592c37bb2,40087,1732486423713 2024-11-24T22:15:28,552 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732486521610.94f3ccf9f8a638fa974e1f69627832ef., hostname=bd6592c37bb2,40087,1732486423713, seqNum=5 , the old value is region=TestAcidGuarantees,,1732486521610.94f3ccf9f8a638fa974e1f69627832ef., hostname=bd6592c37bb2,40087,1732486423713, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94f3ccf9f8a638fa974e1f69627832ef, server=bd6592c37bb2,40087,1732486423713 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T22:15:28,552 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732486521610.94f3ccf9f8a638fa974e1f69627832ef., hostname=bd6592c37bb2,40087,1732486423713, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94f3ccf9f8a638fa974e1f69627832ef, server=bd6592c37bb2,40087,1732486423713 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T22:15:28,552 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732486521610.94f3ccf9f8a638fa974e1f69627832ef., hostname=bd6592c37bb2,40087,1732486423713, seqNum=5 because the exception is null or not the one we care about 2024-11-24T22:15:28,560 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40087 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94f3ccf9f8a638fa974e1f69627832ef, server=bd6592c37bb2,40087,1732486423713 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-24T22:15:28,560 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40087 {}] ipc.CallRunner(138): callId: 24 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:32822 deadline: 1732486588551, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94f3ccf9f8a638fa974e1f69627832ef, server=bd6592c37bb2,40087,1732486423713 2024-11-24T22:15:28,560 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732486521610.94f3ccf9f8a638fa974e1f69627832ef., hostname=bd6592c37bb2,40087,1732486423713, seqNum=5 , the old value is region=TestAcidGuarantees,,1732486521610.94f3ccf9f8a638fa974e1f69627832ef., hostname=bd6592c37bb2,40087,1732486423713, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94f3ccf9f8a638fa974e1f69627832ef, server=bd6592c37bb2,40087,1732486423713 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T22:15:28,560 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732486521610.94f3ccf9f8a638fa974e1f69627832ef., hostname=bd6592c37bb2,40087,1732486423713, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94f3ccf9f8a638fa974e1f69627832ef, server=bd6592c37bb2,40087,1732486423713 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T22:15:28,560 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732486521610.94f3ccf9f8a638fa974e1f69627832ef., hostname=bd6592c37bb2,40087,1732486423713, seqNum=5 because the exception is null or not the one we care about 2024-11-24T22:15:28,568 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40087 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94f3ccf9f8a638fa974e1f69627832ef, server=bd6592c37bb2,40087,1732486423713 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-24T22:15:28,568 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40087 {}] ipc.CallRunner(138): callId: 23 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:32846 deadline: 1732486588561, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94f3ccf9f8a638fa974e1f69627832ef, server=bd6592c37bb2,40087,1732486423713 2024-11-24T22:15:28,568 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40087 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94f3ccf9f8a638fa974e1f69627832ef, server=bd6592c37bb2,40087,1732486423713 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-24T22:15:28,568 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732486521610.94f3ccf9f8a638fa974e1f69627832ef., hostname=bd6592c37bb2,40087,1732486423713, seqNum=5 , the old value is region=TestAcidGuarantees,,1732486521610.94f3ccf9f8a638fa974e1f69627832ef., hostname=bd6592c37bb2,40087,1732486423713, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94f3ccf9f8a638fa974e1f69627832ef, server=bd6592c37bb2,40087,1732486423713 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T22:15:28,568 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732486521610.94f3ccf9f8a638fa974e1f69627832ef., hostname=bd6592c37bb2,40087,1732486423713, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94f3ccf9f8a638fa974e1f69627832ef, server=bd6592c37bb2,40087,1732486423713 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T22:15:28,568 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40087 {}] ipc.CallRunner(138): callId: 24 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:32810 deadline: 1732486588561, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94f3ccf9f8a638fa974e1f69627832ef, server=bd6592c37bb2,40087,1732486423713 2024-11-24T22:15:28,568 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732486521610.94f3ccf9f8a638fa974e1f69627832ef., hostname=bd6592c37bb2,40087,1732486423713, seqNum=5 because the exception is null or not the one we care about 2024-11-24T22:15:28,568 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732486521610.94f3ccf9f8a638fa974e1f69627832ef., hostname=bd6592c37bb2,40087,1732486423713, seqNum=5 , the old value is region=TestAcidGuarantees,,1732486521610.94f3ccf9f8a638fa974e1f69627832ef., hostname=bd6592c37bb2,40087,1732486423713, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94f3ccf9f8a638fa974e1f69627832ef, server=bd6592c37bb2,40087,1732486423713 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T22:15:28,569 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732486521610.94f3ccf9f8a638fa974e1f69627832ef., hostname=bd6592c37bb2,40087,1732486423713, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94f3ccf9f8a638fa974e1f69627832ef, server=bd6592c37bb2,40087,1732486423713 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T22:15:28,569 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732486521610.94f3ccf9f8a638fa974e1f69627832ef., hostname=bd6592c37bb2,40087,1732486423713, seqNum=5 because the exception is null or not the one we care about 2024-11-24T22:15:28,638 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=49 (bloomFilter=true), to=hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/94f3ccf9f8a638fa974e1f69627832ef/.tmp/C/2ba25fa3975e49de8bfb037efe930ba5 2024-11-24T22:15:28,643 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/94f3ccf9f8a638fa974e1f69627832ef/.tmp/A/8ce2903e8a5b46edb21934de4a278e2d as hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/94f3ccf9f8a638fa974e1f69627832ef/A/8ce2903e8a5b46edb21934de4a278e2d 2024-11-24T22:15:28,646 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/94f3ccf9f8a638fa974e1f69627832ef/A/8ce2903e8a5b46edb21934de4a278e2d, entries=200, sequenceid=49, filesize=38.6 K 2024-11-24T22:15:28,647 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/94f3ccf9f8a638fa974e1f69627832ef/.tmp/B/178dfc664e9c455d93e59f22305934cf as hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/94f3ccf9f8a638fa974e1f69627832ef/B/178dfc664e9c455d93e59f22305934cf 2024-11-24T22:15:28,651 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/94f3ccf9f8a638fa974e1f69627832ef/B/178dfc664e9c455d93e59f22305934cf, entries=150, sequenceid=49, filesize=11.7 K 2024-11-24T22:15:28,652 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/94f3ccf9f8a638fa974e1f69627832ef/.tmp/C/2ba25fa3975e49de8bfb037efe930ba5 as hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/94f3ccf9f8a638fa974e1f69627832ef/C/2ba25fa3975e49de8bfb037efe930ba5 2024-11-24T22:15:28,663 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/94f3ccf9f8a638fa974e1f69627832ef/C/2ba25fa3975e49de8bfb037efe930ba5, entries=150, sequenceid=49, filesize=11.7 K 2024-11-24T22:15:28,664 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~67.09 KB/68700, heapSize ~199.92 KB/204720, currentSize=107.34 KB/109920 for 94f3ccf9f8a638fa974e1f69627832ef in 498ms, sequenceid=49, compaction requested=true 2024-11-24T22:15:28,664 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 94f3ccf9f8a638fa974e1f69627832ef: 2024-11-24T22:15:28,664 DEBUG [RS:0;bd6592c37bb2:40087-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-24T22:15:28,664 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 94f3ccf9f8a638fa974e1f69627832ef:A, priority=-2147483648, current under compaction store size is 1 2024-11-24T22:15:28,664 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-24T22:15:28,664 DEBUG [RS:0;bd6592c37bb2:40087-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-24T22:15:28,664 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 94f3ccf9f8a638fa974e1f69627832ef:B, priority=-2147483648, current under compaction store size is 2 2024-11-24T22:15:28,664 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-24T22:15:28,664 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 94f3ccf9f8a638fa974e1f69627832ef:C, priority=-2147483648, current under compaction store size is 3 2024-11-24T22:15:28,664 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-24T22:15:28,665 DEBUG [RS:0;bd6592c37bb2:40087-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 101459 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-24T22:15:28,665 DEBUG [RS:0;bd6592c37bb2:40087-shortCompactions-0 {}] regionserver.HStore(1541): 94f3ccf9f8a638fa974e1f69627832ef/A is initiating minor compaction (all files) 2024-11-24T22:15:28,665 INFO [RS:0;bd6592c37bb2:40087-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 94f3ccf9f8a638fa974e1f69627832ef/A in TestAcidGuarantees,,1732486521610.94f3ccf9f8a638fa974e1f69627832ef. 2024-11-24T22:15:28,665 INFO [RS:0;bd6592c37bb2:40087-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/94f3ccf9f8a638fa974e1f69627832ef/A/a8661fe5f09945f8996df4fc0affd5f9, hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/94f3ccf9f8a638fa974e1f69627832ef/A/eea8340353634d808eee2f348943af0e, hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/94f3ccf9f8a638fa974e1f69627832ef/A/8ce2903e8a5b46edb21934de4a278e2d] into tmpdir=hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/94f3ccf9f8a638fa974e1f69627832ef/.tmp, totalSize=99.1 K 2024-11-24T22:15:28,665 INFO [RS:0;bd6592c37bb2:40087-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=13 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1732486521610.94f3ccf9f8a638fa974e1f69627832ef. 2024-11-24T22:15:28,665 DEBUG [RS:0;bd6592c37bb2:40087-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1732486521610.94f3ccf9f8a638fa974e1f69627832ef. files: [hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/94f3ccf9f8a638fa974e1f69627832ef/A/a8661fe5f09945f8996df4fc0affd5f9, hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/94f3ccf9f8a638fa974e1f69627832ef/A/eea8340353634d808eee2f348943af0e, hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/94f3ccf9f8a638fa974e1f69627832ef/A/8ce2903e8a5b46edb21934de4a278e2d] 2024-11-24T22:15:28,666 DEBUG [RS:0;bd6592c37bb2:40087-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36003 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-24T22:15:28,666 DEBUG [RS:0;bd6592c37bb2:40087-shortCompactions-0 {}] compactions.Compactor(225): Compacting a8661fe5f09945f8996df4fc0affd5f9, keycount=150, bloomtype=ROW, size=30.2 K, encoding=NONE, compression=NONE, seqNum=16, earliestPutTs=1732486524716 2024-11-24T22:15:28,666 DEBUG [RS:0;bd6592c37bb2:40087-longCompactions-0 {}] regionserver.HStore(1541): 94f3ccf9f8a638fa974e1f69627832ef/B is initiating minor compaction (all files) 2024-11-24T22:15:28,666 INFO [RS:0;bd6592c37bb2:40087-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 94f3ccf9f8a638fa974e1f69627832ef/B in TestAcidGuarantees,,1732486521610.94f3ccf9f8a638fa974e1f69627832ef. 2024-11-24T22:15:28,666 INFO [RS:0;bd6592c37bb2:40087-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/94f3ccf9f8a638fa974e1f69627832ef/B/cd94decc27924b9687ddf2ec811ae351, hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/94f3ccf9f8a638fa974e1f69627832ef/B/cc5afda838d84551b7b988a9ea3631f6, hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/94f3ccf9f8a638fa974e1f69627832ef/B/178dfc664e9c455d93e59f22305934cf] into tmpdir=hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/94f3ccf9f8a638fa974e1f69627832ef/.tmp, totalSize=35.2 K 2024-11-24T22:15:28,666 DEBUG [RS:0;bd6592c37bb2:40087-shortCompactions-0 {}] compactions.Compactor(225): Compacting eea8340353634d808eee2f348943af0e, keycount=150, bloomtype=ROW, size=30.2 K, encoding=NONE, compression=NONE, seqNum=36, earliestPutTs=1732486524799 2024-11-24T22:15:28,667 DEBUG [RS:0;bd6592c37bb2:40087-longCompactions-0 {}] compactions.Compactor(225): Compacting cd94decc27924b9687ddf2ec811ae351, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=16, earliestPutTs=1732486524716 2024-11-24T22:15:28,667 DEBUG [RS:0;bd6592c37bb2:40087-shortCompactions-0 {}] compactions.Compactor(225): Compacting 8ce2903e8a5b46edb21934de4a278e2d, keycount=200, bloomtype=ROW, size=38.6 K, encoding=NONE, compression=NONE, seqNum=49, earliestPutTs=1732486526992 2024-11-24T22:15:28,667 DEBUG [RS:0;bd6592c37bb2:40087-longCompactions-0 {}] compactions.Compactor(225): Compacting cc5afda838d84551b7b988a9ea3631f6, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=36, earliestPutTs=1732486524799 2024-11-24T22:15:28,667 DEBUG [RS:0;bd6592c37bb2:40087-longCompactions-0 {}] compactions.Compactor(225): Compacting 178dfc664e9c455d93e59f22305934cf, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=49, earliestPutTs=1732486526999 2024-11-24T22:15:28,674 INFO [RS:0;bd6592c37bb2:40087-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=94f3ccf9f8a638fa974e1f69627832ef] 2024-11-24T22:15:28,675 DEBUG [RS:0;bd6592c37bb2:40087-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e20241124d768b088054e416dbc6a27afad462ee0_94f3ccf9f8a638fa974e1f69627832ef store=[table=TestAcidGuarantees family=A region=94f3ccf9f8a638fa974e1f69627832ef] 2024-11-24T22:15:28,675 INFO [RS:0;bd6592c37bb2:40087-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 94f3ccf9f8a638fa974e1f69627832ef#B#compaction#310 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-24T22:15:28,676 DEBUG [RS:0;bd6592c37bb2:40087-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/94f3ccf9f8a638fa974e1f69627832ef/.tmp/B/e6bb3a30f1f848dd9dea06828499b452 is 50, key is test_row_0/B:col10/1732486526999/Put/seqid=0 2024-11-24T22:15:28,678 DEBUG [RS:0;bd6592c37bb2:40087-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e20241124d768b088054e416dbc6a27afad462ee0_94f3ccf9f8a638fa974e1f69627832ef, store=[table=TestAcidGuarantees family=A region=94f3ccf9f8a638fa974e1f69627832ef] 2024-11-24T22:15:28,678 DEBUG [RS:0;bd6592c37bb2:40087-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241124d768b088054e416dbc6a27afad462ee0_94f3ccf9f8a638fa974e1f69627832ef because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=94f3ccf9f8a638fa974e1f69627832ef] 2024-11-24T22:15:28,684 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45929 is added to blk_1073742190_1366 (size=4469) 2024-11-24T22:15:28,690 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45929 is added to blk_1073742189_1365 (size=12104) 2024-11-24T22:15:28,861 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37701 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=97 2024-11-24T22:15:28,861 INFO [RPCClient-NioEventLoopGroup-4-8 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-24T22:15:28,862 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37701 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-24T22:15:28,862 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37701 {}] procedure2.ProcedureExecutor(1139): Stored pid=99, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=99, table=TestAcidGuarantees 2024-11-24T22:15:28,863 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37701 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=99 2024-11-24T22:15:28,863 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=99, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=99, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-24T22:15:28,864 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=99, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=99, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-24T22:15:28,864 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=100, ppid=99, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-24T22:15:28,868 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40087 {}] regionserver.HRegion(8855): Flush requested on 94f3ccf9f8a638fa974e1f69627832ef 2024-11-24T22:15:28,868 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 94f3ccf9f8a638fa974e1f69627832ef 3/3 column families, dataSize=114.05 KB heapSize=339.42 KB 2024-11-24T22:15:28,868 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 94f3ccf9f8a638fa974e1f69627832ef, store=A 2024-11-24T22:15:28,868 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-24T22:15:28,868 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 94f3ccf9f8a638fa974e1f69627832ef, store=B 2024-11-24T22:15:28,868 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-24T22:15:28,868 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 94f3ccf9f8a638fa974e1f69627832ef, store=C 2024-11-24T22:15:28,868 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-24T22:15:28,875 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241124e4beb5734e154df990570008741226e0_94f3ccf9f8a638fa974e1f69627832ef is 50, key is test_row_0/A:col10/1732486528866/Put/seqid=0 2024-11-24T22:15:28,886 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45929 is added to blk_1073742191_1367 (size=14594) 2024-11-24T22:15:28,886 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T22:15:28,895 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241124e4beb5734e154df990570008741226e0_94f3ccf9f8a638fa974e1f69627832ef to hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241124e4beb5734e154df990570008741226e0_94f3ccf9f8a638fa974e1f69627832ef 2024-11-24T22:15:28,896 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/94f3ccf9f8a638fa974e1f69627832ef/.tmp/A/95ed96d635f54bce8685e4aa528975e7, store: [table=TestAcidGuarantees family=A region=94f3ccf9f8a638fa974e1f69627832ef] 2024-11-24T22:15:28,896 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/94f3ccf9f8a638fa974e1f69627832ef/.tmp/A/95ed96d635f54bce8685e4aa528975e7 is 175, key is test_row_0/A:col10/1732486528866/Put/seqid=0 2024-11-24T22:15:28,899 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45929 is added to blk_1073742192_1368 (size=39549) 2024-11-24T22:15:28,900 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=69, memsize=38.0 K, hasBloomFilter=true, into tmp file hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/94f3ccf9f8a638fa974e1f69627832ef/.tmp/A/95ed96d635f54bce8685e4aa528975e7 2024-11-24T22:15:28,905 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40087 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94f3ccf9f8a638fa974e1f69627832ef, server=bd6592c37bb2,40087,1732486423713 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-24T22:15:28,905 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40087 {}] ipc.CallRunner(138): callId: 26 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:32810 deadline: 1732486588889, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94f3ccf9f8a638fa974e1f69627832ef, server=bd6592c37bb2,40087,1732486423713 2024-11-24T22:15:28,906 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40087 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94f3ccf9f8a638fa974e1f69627832ef, server=bd6592c37bb2,40087,1732486423713 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-24T22:15:28,906 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40087 {}] ipc.CallRunner(138): callId: 29 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:32774 deadline: 1732486588894, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94f3ccf9f8a638fa974e1f69627832ef, server=bd6592c37bb2,40087,1732486423713 2024-11-24T22:15:28,906 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732486521610.94f3ccf9f8a638fa974e1f69627832ef., hostname=bd6592c37bb2,40087,1732486423713, seqNum=5 , the old value is region=TestAcidGuarantees,,1732486521610.94f3ccf9f8a638fa974e1f69627832ef., hostname=bd6592c37bb2,40087,1732486423713, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94f3ccf9f8a638fa974e1f69627832ef, server=bd6592c37bb2,40087,1732486423713 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T22:15:28,906 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732486521610.94f3ccf9f8a638fa974e1f69627832ef., hostname=bd6592c37bb2,40087,1732486423713, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94f3ccf9f8a638fa974e1f69627832ef, server=bd6592c37bb2,40087,1732486423713 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T22:15:28,906 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732486521610.94f3ccf9f8a638fa974e1f69627832ef., hostname=bd6592c37bb2,40087,1732486423713, seqNum=5 because the exception is null or not the one we care about 2024-11-24T22:15:28,906 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/94f3ccf9f8a638fa974e1f69627832ef/.tmp/B/26545f645a554d1daddca6dbf2adc469 is 50, key is test_row_0/B:col10/1732486528866/Put/seqid=0 2024-11-24T22:15:28,906 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732486521610.94f3ccf9f8a638fa974e1f69627832ef., hostname=bd6592c37bb2,40087,1732486423713, seqNum=5 , the old value is region=TestAcidGuarantees,,1732486521610.94f3ccf9f8a638fa974e1f69627832ef., hostname=bd6592c37bb2,40087,1732486423713, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94f3ccf9f8a638fa974e1f69627832ef, server=bd6592c37bb2,40087,1732486423713 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T22:15:28,906 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732486521610.94f3ccf9f8a638fa974e1f69627832ef., hostname=bd6592c37bb2,40087,1732486423713, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94f3ccf9f8a638fa974e1f69627832ef, server=bd6592c37bb2,40087,1732486423713 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T22:15:28,906 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732486521610.94f3ccf9f8a638fa974e1f69627832ef., hostname=bd6592c37bb2,40087,1732486423713, seqNum=5 because the exception is null or not the one we care about 2024-11-24T22:15:28,909 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40087 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94f3ccf9f8a638fa974e1f69627832ef, server=bd6592c37bb2,40087,1732486423713 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-24T22:15:28,909 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40087 {}] ipc.CallRunner(138): callId: 28 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:32822 deadline: 1732486588896, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94f3ccf9f8a638fa974e1f69627832ef, server=bd6592c37bb2,40087,1732486423713 2024-11-24T22:15:28,909 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45929 is added to blk_1073742193_1369 (size=12001) 2024-11-24T22:15:28,910 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=38.02 KB at sequenceid=69 (bloomFilter=true), to=hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/94f3ccf9f8a638fa974e1f69627832ef/.tmp/B/26545f645a554d1daddca6dbf2adc469 2024-11-24T22:15:28,911 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732486521610.94f3ccf9f8a638fa974e1f69627832ef., hostname=bd6592c37bb2,40087,1732486423713, seqNum=5 , the old value is region=TestAcidGuarantees,,1732486521610.94f3ccf9f8a638fa974e1f69627832ef., hostname=bd6592c37bb2,40087,1732486423713, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94f3ccf9f8a638fa974e1f69627832ef, server=bd6592c37bb2,40087,1732486423713 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T22:15:28,911 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732486521610.94f3ccf9f8a638fa974e1f69627832ef., hostname=bd6592c37bb2,40087,1732486423713, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94f3ccf9f8a638fa974e1f69627832ef, server=bd6592c37bb2,40087,1732486423713 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T22:15:28,911 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732486521610.94f3ccf9f8a638fa974e1f69627832ef., hostname=bd6592c37bb2,40087,1732486423713, seqNum=5 because the exception is null or not the one we care about 2024-11-24T22:15:28,915 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40087 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94f3ccf9f8a638fa974e1f69627832ef, server=bd6592c37bb2,40087,1732486423713 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-24T22:15:28,915 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40087 {}] ipc.CallRunner(138): callId: 26 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:32846 deadline: 1732486588906, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94f3ccf9f8a638fa974e1f69627832ef, server=bd6592c37bb2,40087,1732486423713 2024-11-24T22:15:28,915 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732486521610.94f3ccf9f8a638fa974e1f69627832ef., hostname=bd6592c37bb2,40087,1732486423713, seqNum=5 , the old value is region=TestAcidGuarantees,,1732486521610.94f3ccf9f8a638fa974e1f69627832ef., hostname=bd6592c37bb2,40087,1732486423713, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94f3ccf9f8a638fa974e1f69627832ef, server=bd6592c37bb2,40087,1732486423713 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T22:15:28,915 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732486521610.94f3ccf9f8a638fa974e1f69627832ef., hostname=bd6592c37bb2,40087,1732486423713, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94f3ccf9f8a638fa974e1f69627832ef, server=bd6592c37bb2,40087,1732486423713 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T22:15:28,916 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732486521610.94f3ccf9f8a638fa974e1f69627832ef., hostname=bd6592c37bb2,40087,1732486423713, seqNum=5 because the exception is null or not the one we care about 2024-11-24T22:15:28,917 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/94f3ccf9f8a638fa974e1f69627832ef/.tmp/C/e003126d696c418db6a92bf6d9e16671 is 50, key is test_row_0/C:col10/1732486528866/Put/seqid=0 2024-11-24T22:15:28,921 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45929 is added to blk_1073742194_1370 (size=12001) 2024-11-24T22:15:28,925 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=38.02 KB at sequenceid=69 (bloomFilter=true), to=hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/94f3ccf9f8a638fa974e1f69627832ef/.tmp/C/e003126d696c418db6a92bf6d9e16671 2024-11-24T22:15:28,930 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/94f3ccf9f8a638fa974e1f69627832ef/.tmp/A/95ed96d635f54bce8685e4aa528975e7 as hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/94f3ccf9f8a638fa974e1f69627832ef/A/95ed96d635f54bce8685e4aa528975e7 2024-11-24T22:15:28,933 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/94f3ccf9f8a638fa974e1f69627832ef/A/95ed96d635f54bce8685e4aa528975e7, entries=200, sequenceid=69, filesize=38.6 K 2024-11-24T22:15:28,934 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/94f3ccf9f8a638fa974e1f69627832ef/.tmp/B/26545f645a554d1daddca6dbf2adc469 as hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/94f3ccf9f8a638fa974e1f69627832ef/B/26545f645a554d1daddca6dbf2adc469 2024-11-24T22:15:28,938 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/94f3ccf9f8a638fa974e1f69627832ef/B/26545f645a554d1daddca6dbf2adc469, entries=150, sequenceid=69, filesize=11.7 K 2024-11-24T22:15:28,939 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/94f3ccf9f8a638fa974e1f69627832ef/.tmp/C/e003126d696c418db6a92bf6d9e16671 as hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/94f3ccf9f8a638fa974e1f69627832ef/C/e003126d696c418db6a92bf6d9e16671 2024-11-24T22:15:28,943 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/94f3ccf9f8a638fa974e1f69627832ef/C/e003126d696c418db6a92bf6d9e16671, entries=150, sequenceid=69, filesize=11.7 K 2024-11-24T22:15:28,943 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~114.05 KB/116790, heapSize ~339.38 KB/347520, currentSize=60.38 KB/61830 for 94f3ccf9f8a638fa974e1f69627832ef in 75ms, sequenceid=69, compaction requested=true 2024-11-24T22:15:28,943 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 94f3ccf9f8a638fa974e1f69627832ef: 2024-11-24T22:15:28,944 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 94f3ccf9f8a638fa974e1f69627832ef:A, priority=-2147483648, current under compaction store size is 3 2024-11-24T22:15:28,944 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=2), splitQueue=0 2024-11-24T22:15:28,944 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 94f3ccf9f8a638fa974e1f69627832ef:B, priority=-2147483648, current under compaction store size is 3 2024-11-24T22:15:28,944 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=3), splitQueue=0 2024-11-24T22:15:28,944 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 94f3ccf9f8a638fa974e1f69627832ef:C, priority=-2147483648, current under compaction store size is 3 2024-11-24T22:15:28,944 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=4), splitQueue=0 2024-11-24T22:15:28,970 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37701 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=99 2024-11-24T22:15:29,015 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40087 {}] regionserver.HRegion(8855): Flush requested on 94f3ccf9f8a638fa974e1f69627832ef 2024-11-24T22:15:29,015 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 94f3ccf9f8a638fa974e1f69627832ef 3/3 column families, dataSize=67.09 KB heapSize=199.97 KB 2024-11-24T22:15:29,016 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 94f3ccf9f8a638fa974e1f69627832ef, store=A 2024-11-24T22:15:29,016 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-24T22:15:29,016 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 94f3ccf9f8a638fa974e1f69627832ef, store=B 2024-11-24T22:15:29,016 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-24T22:15:29,016 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 94f3ccf9f8a638fa974e1f69627832ef, store=C 2024-11-24T22:15:29,016 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-24T22:15:29,016 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=40087 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=100 2024-11-24T22:15:29,017 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bd6592c37bb2:0-1 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732486521610.94f3ccf9f8a638fa974e1f69627832ef. 2024-11-24T22:15:29,017 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bd6592c37bb2:0-1 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732486521610.94f3ccf9f8a638fa974e1f69627832ef. as already flushing 2024-11-24T22:15:29,017 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bd6592c37bb2:0-1 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732486521610.94f3ccf9f8a638fa974e1f69627832ef. 2024-11-24T22:15:29,017 ERROR [RS_FLUSH_OPERATIONS-regionserver/bd6592c37bb2:0-1 {event_type=RS_FLUSH_REGIONS, pid=100}] handler.RSProcedureHandler(58): pid=100 java.io.IOException: Unable to complete flush {ENCODED => 94f3ccf9f8a638fa974e1f69627832ef, NAME => 'TestAcidGuarantees,,1732486521610.94f3ccf9f8a638fa974e1f69627832ef.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T22:15:29,017 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bd6592c37bb2:0-1 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=100 java.io.IOException: Unable to complete flush {ENCODED => 94f3ccf9f8a638fa974e1f69627832ef, NAME => 'TestAcidGuarantees,,1732486521610.94f3ccf9f8a638fa974e1f69627832ef.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T22:15:29,018 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37701 {}] master.HMaster(4177): Remote procedure failed, pid=100 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 94f3ccf9f8a638fa974e1f69627832ef, NAME => 'TestAcidGuarantees,,1732486521610.94f3ccf9f8a638fa974e1f69627832ef.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 94f3ccf9f8a638fa974e1f69627832ef, NAME => 'TestAcidGuarantees,,1732486521610.94f3ccf9f8a638fa974e1f69627832ef.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T22:15:29,023 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241124cc8a4013fb4f4d0780b44c16d7bd24b5_94f3ccf9f8a638fa974e1f69627832ef is 50, key is test_row_0/A:col10/1732486528887/Put/seqid=0 2024-11-24T22:15:29,027 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45929 is added to blk_1073742195_1371 (size=14594) 2024-11-24T22:15:29,069 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40087 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94f3ccf9f8a638fa974e1f69627832ef, server=bd6592c37bb2,40087,1732486423713 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-24T22:15:29,069 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40087 {}] ipc.CallRunner(138): callId: 34 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:32774 deadline: 1732486589052, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94f3ccf9f8a638fa974e1f69627832ef, server=bd6592c37bb2,40087,1732486423713 2024-11-24T22:15:29,069 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732486521610.94f3ccf9f8a638fa974e1f69627832ef., hostname=bd6592c37bb2,40087,1732486423713, seqNum=5 , the old value is region=TestAcidGuarantees,,1732486521610.94f3ccf9f8a638fa974e1f69627832ef., hostname=bd6592c37bb2,40087,1732486423713, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94f3ccf9f8a638fa974e1f69627832ef, server=bd6592c37bb2,40087,1732486423713 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T22:15:29,069 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732486521610.94f3ccf9f8a638fa974e1f69627832ef., hostname=bd6592c37bb2,40087,1732486423713, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94f3ccf9f8a638fa974e1f69627832ef, server=bd6592c37bb2,40087,1732486423713 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T22:15:29,070 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732486521610.94f3ccf9f8a638fa974e1f69627832ef., hostname=bd6592c37bb2,40087,1732486423713, seqNum=5 because the exception is null or not the one we care about 2024-11-24T22:15:29,070 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40087 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94f3ccf9f8a638fa974e1f69627832ef, server=bd6592c37bb2,40087,1732486423713 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-24T22:15:29,070 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40087 {}] ipc.CallRunner(138): callId: 31 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:32810 deadline: 1732486589053, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94f3ccf9f8a638fa974e1f69627832ef, server=bd6592c37bb2,40087,1732486423713 2024-11-24T22:15:29,070 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40087 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94f3ccf9f8a638fa974e1f69627832ef, server=bd6592c37bb2,40087,1732486423713 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-24T22:15:29,070 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732486521610.94f3ccf9f8a638fa974e1f69627832ef., hostname=bd6592c37bb2,40087,1732486423713, seqNum=5 , the old value is region=TestAcidGuarantees,,1732486521610.94f3ccf9f8a638fa974e1f69627832ef., hostname=bd6592c37bb2,40087,1732486423713, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94f3ccf9f8a638fa974e1f69627832ef, server=bd6592c37bb2,40087,1732486423713 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T22:15:29,070 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40087 {}] ipc.CallRunner(138): callId: 30 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:32846 deadline: 1732486589054, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94f3ccf9f8a638fa974e1f69627832ef, server=bd6592c37bb2,40087,1732486423713 2024-11-24T22:15:29,070 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732486521610.94f3ccf9f8a638fa974e1f69627832ef., hostname=bd6592c37bb2,40087,1732486423713, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94f3ccf9f8a638fa974e1f69627832ef, server=bd6592c37bb2,40087,1732486423713 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T22:15:29,070 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732486521610.94f3ccf9f8a638fa974e1f69627832ef., hostname=bd6592c37bb2,40087,1732486423713, seqNum=5 because the exception is null or not the one we care about 2024-11-24T22:15:29,070 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40087 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94f3ccf9f8a638fa974e1f69627832ef, server=bd6592c37bb2,40087,1732486423713 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-24T22:15:29,071 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732486521610.94f3ccf9f8a638fa974e1f69627832ef., hostname=bd6592c37bb2,40087,1732486423713, seqNum=5 , the old value is region=TestAcidGuarantees,,1732486521610.94f3ccf9f8a638fa974e1f69627832ef., hostname=bd6592c37bb2,40087,1732486423713, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94f3ccf9f8a638fa974e1f69627832ef, server=bd6592c37bb2,40087,1732486423713 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T22:15:29,071 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40087 {}] ipc.CallRunner(138): callId: 15 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:32832 deadline: 1732486589068, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94f3ccf9f8a638fa974e1f69627832ef, server=bd6592c37bb2,40087,1732486423713 2024-11-24T22:15:29,071 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732486521610.94f3ccf9f8a638fa974e1f69627832ef., hostname=bd6592c37bb2,40087,1732486423713, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94f3ccf9f8a638fa974e1f69627832ef, server=bd6592c37bb2,40087,1732486423713 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T22:15:29,071 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732486521610.94f3ccf9f8a638fa974e1f69627832ef., hostname=bd6592c37bb2,40087,1732486423713, seqNum=5 because the exception is null or not the one we care about 2024-11-24T22:15:29,071 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732486521610.94f3ccf9f8a638fa974e1f69627832ef., hostname=bd6592c37bb2,40087,1732486423713, seqNum=5 , the old value is region=TestAcidGuarantees,,1732486521610.94f3ccf9f8a638fa974e1f69627832ef., hostname=bd6592c37bb2,40087,1732486423713, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94f3ccf9f8a638fa974e1f69627832ef, server=bd6592c37bb2,40087,1732486423713 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T22:15:29,071 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732486521610.94f3ccf9f8a638fa974e1f69627832ef., hostname=bd6592c37bb2,40087,1732486423713, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94f3ccf9f8a638fa974e1f69627832ef, server=bd6592c37bb2,40087,1732486423713 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T22:15:29,071 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732486521610.94f3ccf9f8a638fa974e1f69627832ef., hostname=bd6592c37bb2,40087,1732486423713, seqNum=5 because the exception is null or not the one we care about 2024-11-24T22:15:29,085 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40087 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94f3ccf9f8a638fa974e1f69627832ef, server=bd6592c37bb2,40087,1732486423713 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-24T22:15:29,085 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40087 {}] ipc.CallRunner(138): callId: 32 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:32822 deadline: 1732486589069, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94f3ccf9f8a638fa974e1f69627832ef, server=bd6592c37bb2,40087,1732486423713 2024-11-24T22:15:29,085 INFO [RS:0;bd6592c37bb2:40087-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 94f3ccf9f8a638fa974e1f69627832ef#A#compaction#309 average throughput is 0.06 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-24T22:15:29,086 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732486521610.94f3ccf9f8a638fa974e1f69627832ef., hostname=bd6592c37bb2,40087,1732486423713, seqNum=5 , the old value is region=TestAcidGuarantees,,1732486521610.94f3ccf9f8a638fa974e1f69627832ef., hostname=bd6592c37bb2,40087,1732486423713, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94f3ccf9f8a638fa974e1f69627832ef, server=bd6592c37bb2,40087,1732486423713 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T22:15:29,086 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732486521610.94f3ccf9f8a638fa974e1f69627832ef., hostname=bd6592c37bb2,40087,1732486423713, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94f3ccf9f8a638fa974e1f69627832ef, server=bd6592c37bb2,40087,1732486423713 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T22:15:29,086 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732486521610.94f3ccf9f8a638fa974e1f69627832ef., hostname=bd6592c37bb2,40087,1732486423713, seqNum=5 because the exception is null or not the one we care about 2024-11-24T22:15:29,086 DEBUG [RS:0;bd6592c37bb2:40087-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/94f3ccf9f8a638fa974e1f69627832ef/.tmp/A/1b69565af4094bd79b4d460cb7671894 is 175, key is test_row_0/A:col10/1732486526999/Put/seqid=0 2024-11-24T22:15:29,091 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45929 is added to blk_1073742196_1372 (size=31058) 2024-11-24T22:15:29,094 DEBUG [RS:0;bd6592c37bb2:40087-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/94f3ccf9f8a638fa974e1f69627832ef/.tmp/B/e6bb3a30f1f848dd9dea06828499b452 as hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/94f3ccf9f8a638fa974e1f69627832ef/B/e6bb3a30f1f848dd9dea06828499b452 2024-11-24T22:15:29,097 INFO [RS:0;bd6592c37bb2:40087-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 94f3ccf9f8a638fa974e1f69627832ef/B of 94f3ccf9f8a638fa974e1f69627832ef into e6bb3a30f1f848dd9dea06828499b452(size=11.8 K), total size for store is 23.5 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-24T22:15:29,097 DEBUG [RS:0;bd6592c37bb2:40087-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 94f3ccf9f8a638fa974e1f69627832ef: 2024-11-24T22:15:29,098 INFO [RS:0;bd6592c37bb2:40087-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732486521610.94f3ccf9f8a638fa974e1f69627832ef., storeName=94f3ccf9f8a638fa974e1f69627832ef/B, priority=13, startTime=1732486528664; duration=0sec 2024-11-24T22:15:29,098 DEBUG [RS:0;bd6592c37bb2:40087-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=4), splitQueue=0 2024-11-24T22:15:29,098 DEBUG [RS:0;bd6592c37bb2:40087-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 94f3ccf9f8a638fa974e1f69627832ef:B 2024-11-24T22:15:29,098 DEBUG [RS:0;bd6592c37bb2:40087-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 3 compacting, 1 eligible, 16 blocking 2024-11-24T22:15:29,098 DEBUG [RS:0;bd6592c37bb2:40087-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 0 files of size 0 starting at candidate #-1 after considering 0 permutations with 0 in ratio 2024-11-24T22:15:29,099 DEBUG [RS:0;bd6592c37bb2:40087-longCompactions-0 {}] compactions.SortedCompactionPolicy(232): Not compacting files because we only have 0 files ready for compaction. Need 3 to initiate. 2024-11-24T22:15:29,099 DEBUG [RS:0;bd6592c37bb2:40087-longCompactions-0 {}] regionserver.CompactSplit(450): Not compacting TestAcidGuarantees,,1732486521610.94f3ccf9f8a638fa974e1f69627832ef. because compaction request was cancelled 2024-11-24T22:15:29,099 DEBUG [RS:0;bd6592c37bb2:40087-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 94f3ccf9f8a638fa974e1f69627832ef:A 2024-11-24T22:15:29,099 DEBUG [RS:0;bd6592c37bb2:40087-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-24T22:15:29,100 DEBUG [RS:0;bd6592c37bb2:40087-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 48004 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-24T22:15:29,100 DEBUG [RS:0;bd6592c37bb2:40087-longCompactions-0 {}] regionserver.HStore(1541): 94f3ccf9f8a638fa974e1f69627832ef/C is initiating minor compaction (all files) 2024-11-24T22:15:29,100 INFO [RS:0;bd6592c37bb2:40087-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 94f3ccf9f8a638fa974e1f69627832ef/C in TestAcidGuarantees,,1732486521610.94f3ccf9f8a638fa974e1f69627832ef. 2024-11-24T22:15:29,100 INFO [RS:0;bd6592c37bb2:40087-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/94f3ccf9f8a638fa974e1f69627832ef/C/0c6077b470f14406ad8905d5895c98ee, hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/94f3ccf9f8a638fa974e1f69627832ef/C/a6f6fe973984491a9e1d7cecd110be45, hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/94f3ccf9f8a638fa974e1f69627832ef/C/2ba25fa3975e49de8bfb037efe930ba5, hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/94f3ccf9f8a638fa974e1f69627832ef/C/e003126d696c418db6a92bf6d9e16671] into tmpdir=hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/94f3ccf9f8a638fa974e1f69627832ef/.tmp, totalSize=46.9 K 2024-11-24T22:15:29,100 DEBUG [RS:0;bd6592c37bb2:40087-longCompactions-0 {}] compactions.Compactor(225): Compacting 0c6077b470f14406ad8905d5895c98ee, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=16, earliestPutTs=1732486524716 2024-11-24T22:15:29,100 DEBUG [RS:0;bd6592c37bb2:40087-longCompactions-0 {}] compactions.Compactor(225): Compacting a6f6fe973984491a9e1d7cecd110be45, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=36, earliestPutTs=1732486524799 2024-11-24T22:15:29,101 DEBUG [RS:0;bd6592c37bb2:40087-longCompactions-0 {}] compactions.Compactor(225): Compacting 2ba25fa3975e49de8bfb037efe930ba5, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=49, earliestPutTs=1732486526999 2024-11-24T22:15:29,101 DEBUG [RS:0;bd6592c37bb2:40087-longCompactions-0 {}] compactions.Compactor(225): Compacting e003126d696c418db6a92bf6d9e16671, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=69, earliestPutTs=1732486528203 2024-11-24T22:15:29,109 INFO [RS:0;bd6592c37bb2:40087-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 94f3ccf9f8a638fa974e1f69627832ef#C#compaction#315 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-24T22:15:29,109 DEBUG [RS:0;bd6592c37bb2:40087-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/94f3ccf9f8a638fa974e1f69627832ef/.tmp/C/267a2d67eb91491c826a9d1801fe9d29 is 50, key is test_row_0/C:col10/1732486528866/Put/seqid=0 2024-11-24T22:15:29,112 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45929 is added to blk_1073742197_1373 (size=12139) 2024-11-24T22:15:29,170 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=40087 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=100 2024-11-24T22:15:29,170 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bd6592c37bb2:0-2 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732486521610.94f3ccf9f8a638fa974e1f69627832ef. 2024-11-24T22:15:29,170 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bd6592c37bb2:0-2 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732486521610.94f3ccf9f8a638fa974e1f69627832ef. as already flushing 2024-11-24T22:15:29,170 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bd6592c37bb2:0-2 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732486521610.94f3ccf9f8a638fa974e1f69627832ef. 2024-11-24T22:15:29,170 ERROR [RS_FLUSH_OPERATIONS-regionserver/bd6592c37bb2:0-2 {event_type=RS_FLUSH_REGIONS, pid=100}] handler.RSProcedureHandler(58): pid=100 java.io.IOException: Unable to complete flush {ENCODED => 94f3ccf9f8a638fa974e1f69627832ef, NAME => 'TestAcidGuarantees,,1732486521610.94f3ccf9f8a638fa974e1f69627832ef.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T22:15:29,170 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bd6592c37bb2:0-2 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=100 java.io.IOException: Unable to complete flush {ENCODED => 94f3ccf9f8a638fa974e1f69627832ef, NAME => 'TestAcidGuarantees,,1732486521610.94f3ccf9f8a638fa974e1f69627832ef.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T22:15:29,171 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37701 {}] master.HMaster(4177): Remote procedure failed, pid=100 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 94f3ccf9f8a638fa974e1f69627832ef, NAME => 'TestAcidGuarantees,,1732486521610.94f3ccf9f8a638fa974e1f69627832ef.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 94f3ccf9f8a638fa974e1f69627832ef, NAME => 'TestAcidGuarantees,,1732486521610.94f3ccf9f8a638fa974e1f69627832ef.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T22:15:29,181 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37701 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=99 2024-11-24T22:15:29,188 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40087 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94f3ccf9f8a638fa974e1f69627832ef, server=bd6592c37bb2,40087,1732486423713 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-24T22:15:29,189 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40087 {}] ipc.CallRunner(138): callId: 32 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:32810 deadline: 1732486589181, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94f3ccf9f8a638fa974e1f69627832ef, server=bd6592c37bb2,40087,1732486423713 2024-11-24T22:15:29,189 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732486521610.94f3ccf9f8a638fa974e1f69627832ef., hostname=bd6592c37bb2,40087,1732486423713, seqNum=5 , the old value is region=TestAcidGuarantees,,1732486521610.94f3ccf9f8a638fa974e1f69627832ef., hostname=bd6592c37bb2,40087,1732486423713, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94f3ccf9f8a638fa974e1f69627832ef, server=bd6592c37bb2,40087,1732486423713 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T22:15:29,189 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732486521610.94f3ccf9f8a638fa974e1f69627832ef., hostname=bd6592c37bb2,40087,1732486423713, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94f3ccf9f8a638fa974e1f69627832ef, server=bd6592c37bb2,40087,1732486423713 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T22:15:29,189 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40087 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94f3ccf9f8a638fa974e1f69627832ef, server=bd6592c37bb2,40087,1732486423713 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-24T22:15:29,189 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732486521610.94f3ccf9f8a638fa974e1f69627832ef., hostname=bd6592c37bb2,40087,1732486423713, seqNum=5 because the exception is null or not the one we care about 2024-11-24T22:15:29,189 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40087 {}] ipc.CallRunner(138): callId: 16 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:32832 deadline: 1732486589181, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94f3ccf9f8a638fa974e1f69627832ef, server=bd6592c37bb2,40087,1732486423713 2024-11-24T22:15:29,189 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732486521610.94f3ccf9f8a638fa974e1f69627832ef., hostname=bd6592c37bb2,40087,1732486423713, seqNum=5 , the old value is region=TestAcidGuarantees,,1732486521610.94f3ccf9f8a638fa974e1f69627832ef., hostname=bd6592c37bb2,40087,1732486423713, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94f3ccf9f8a638fa974e1f69627832ef, server=bd6592c37bb2,40087,1732486423713 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T22:15:29,189 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40087 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94f3ccf9f8a638fa974e1f69627832ef, server=bd6592c37bb2,40087,1732486423713 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-24T22:15:29,189 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732486521610.94f3ccf9f8a638fa974e1f69627832ef., hostname=bd6592c37bb2,40087,1732486423713, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94f3ccf9f8a638fa974e1f69627832ef, server=bd6592c37bb2,40087,1732486423713 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T22:15:29,189 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732486521610.94f3ccf9f8a638fa974e1f69627832ef., hostname=bd6592c37bb2,40087,1732486423713, seqNum=5 because the exception is null or not the one we care about 2024-11-24T22:15:29,190 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40087 {}] ipc.CallRunner(138): callId: 31 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:32846 deadline: 1732486589181, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94f3ccf9f8a638fa974e1f69627832ef, server=bd6592c37bb2,40087,1732486423713 2024-11-24T22:15:29,190 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732486521610.94f3ccf9f8a638fa974e1f69627832ef., hostname=bd6592c37bb2,40087,1732486423713, seqNum=5 , the old value is region=TestAcidGuarantees,,1732486521610.94f3ccf9f8a638fa974e1f69627832ef., hostname=bd6592c37bb2,40087,1732486423713, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94f3ccf9f8a638fa974e1f69627832ef, server=bd6592c37bb2,40087,1732486423713 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T22:15:29,190 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40087 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94f3ccf9f8a638fa974e1f69627832ef, server=bd6592c37bb2,40087,1732486423713 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-24T22:15:29,190 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732486521610.94f3ccf9f8a638fa974e1f69627832ef., hostname=bd6592c37bb2,40087,1732486423713, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94f3ccf9f8a638fa974e1f69627832ef, server=bd6592c37bb2,40087,1732486423713 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T22:15:29,190 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732486521610.94f3ccf9f8a638fa974e1f69627832ef., hostname=bd6592c37bb2,40087,1732486423713, seqNum=5 because the exception is null or not the one we care about 2024-11-24T22:15:29,190 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40087 {}] ipc.CallRunner(138): callId: 35 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:32774 deadline: 1732486589182, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94f3ccf9f8a638fa974e1f69627832ef, server=bd6592c37bb2,40087,1732486423713 2024-11-24T22:15:29,190 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732486521610.94f3ccf9f8a638fa974e1f69627832ef., hostname=bd6592c37bb2,40087,1732486423713, seqNum=5 , the old value is region=TestAcidGuarantees,,1732486521610.94f3ccf9f8a638fa974e1f69627832ef., hostname=bd6592c37bb2,40087,1732486423713, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94f3ccf9f8a638fa974e1f69627832ef, server=bd6592c37bb2,40087,1732486423713 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T22:15:29,190 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732486521610.94f3ccf9f8a638fa974e1f69627832ef., hostname=bd6592c37bb2,40087,1732486423713, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94f3ccf9f8a638fa974e1f69627832ef, server=bd6592c37bb2,40087,1732486423713 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T22:15:29,190 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732486521610.94f3ccf9f8a638fa974e1f69627832ef., hostname=bd6592c37bb2,40087,1732486423713, seqNum=5 because the exception is null or not the one we care about 2024-11-24T22:15:29,210 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40087 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94f3ccf9f8a638fa974e1f69627832ef, server=bd6592c37bb2,40087,1732486423713 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-24T22:15:29,210 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40087 {}] ipc.CallRunner(138): callId: 33 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:32822 deadline: 1732486589190, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94f3ccf9f8a638fa974e1f69627832ef, server=bd6592c37bb2,40087,1732486423713 2024-11-24T22:15:29,210 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732486521610.94f3ccf9f8a638fa974e1f69627832ef., hostname=bd6592c37bb2,40087,1732486423713, seqNum=5 , the old value is region=TestAcidGuarantees,,1732486521610.94f3ccf9f8a638fa974e1f69627832ef., hostname=bd6592c37bb2,40087,1732486423713, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94f3ccf9f8a638fa974e1f69627832ef, server=bd6592c37bb2,40087,1732486423713 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T22:15:29,210 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732486521610.94f3ccf9f8a638fa974e1f69627832ef., hostname=bd6592c37bb2,40087,1732486423713, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94f3ccf9f8a638fa974e1f69627832ef, server=bd6592c37bb2,40087,1732486423713 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T22:15:29,210 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732486521610.94f3ccf9f8a638fa974e1f69627832ef., hostname=bd6592c37bb2,40087,1732486423713, seqNum=5 because the exception is null or not the one we care about 2024-11-24T22:15:29,322 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=40087 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=100 2024-11-24T22:15:29,323 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bd6592c37bb2:0-0 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732486521610.94f3ccf9f8a638fa974e1f69627832ef. 2024-11-24T22:15:29,323 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bd6592c37bb2:0-0 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732486521610.94f3ccf9f8a638fa974e1f69627832ef. as already flushing 2024-11-24T22:15:29,323 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bd6592c37bb2:0-0 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732486521610.94f3ccf9f8a638fa974e1f69627832ef. 2024-11-24T22:15:29,323 ERROR [RS_FLUSH_OPERATIONS-regionserver/bd6592c37bb2:0-0 {event_type=RS_FLUSH_REGIONS, pid=100}] handler.RSProcedureHandler(58): pid=100 java.io.IOException: Unable to complete flush {ENCODED => 94f3ccf9f8a638fa974e1f69627832ef, NAME => 'TestAcidGuarantees,,1732486521610.94f3ccf9f8a638fa974e1f69627832ef.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T22:15:29,323 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bd6592c37bb2:0-0 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=100 java.io.IOException: Unable to complete flush {ENCODED => 94f3ccf9f8a638fa974e1f69627832ef, NAME => 'TestAcidGuarantees,,1732486521610.94f3ccf9f8a638fa974e1f69627832ef.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T22:15:29,324 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37701 {}] master.HMaster(4177): Remote procedure failed, pid=100 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 94f3ccf9f8a638fa974e1f69627832ef, NAME => 'TestAcidGuarantees,,1732486521610.94f3ccf9f8a638fa974e1f69627832ef.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 94f3ccf9f8a638fa974e1f69627832ef, NAME => 'TestAcidGuarantees,,1732486521610.94f3ccf9f8a638fa974e1f69627832ef.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T22:15:29,400 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40087 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94f3ccf9f8a638fa974e1f69627832ef, server=bd6592c37bb2,40087,1732486423713 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-24T22:15:29,400 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40087 {}] ipc.CallRunner(138): callId: 33 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:32810 deadline: 1732486589390, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94f3ccf9f8a638fa974e1f69627832ef, server=bd6592c37bb2,40087,1732486423713 2024-11-24T22:15:29,401 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732486521610.94f3ccf9f8a638fa974e1f69627832ef., hostname=bd6592c37bb2,40087,1732486423713, seqNum=5 , the old value is region=TestAcidGuarantees,,1732486521610.94f3ccf9f8a638fa974e1f69627832ef., hostname=bd6592c37bb2,40087,1732486423713, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94f3ccf9f8a638fa974e1f69627832ef, server=bd6592c37bb2,40087,1732486423713 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T22:15:29,401 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732486521610.94f3ccf9f8a638fa974e1f69627832ef., hostname=bd6592c37bb2,40087,1732486423713, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94f3ccf9f8a638fa974e1f69627832ef, server=bd6592c37bb2,40087,1732486423713 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T22:15:29,401 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732486521610.94f3ccf9f8a638fa974e1f69627832ef., hostname=bd6592c37bb2,40087,1732486423713, seqNum=5 because the exception is null or not the one we care about 2024-11-24T22:15:29,412 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40087 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94f3ccf9f8a638fa974e1f69627832ef, server=bd6592c37bb2,40087,1732486423713 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-24T22:15:29,413 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40087 {}] ipc.CallRunner(138): callId: 17 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:32832 deadline: 1732486589401, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94f3ccf9f8a638fa974e1f69627832ef, server=bd6592c37bb2,40087,1732486423713 2024-11-24T22:15:29,413 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40087 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94f3ccf9f8a638fa974e1f69627832ef, server=bd6592c37bb2,40087,1732486423713 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-24T22:15:29,413 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732486521610.94f3ccf9f8a638fa974e1f69627832ef., hostname=bd6592c37bb2,40087,1732486423713, seqNum=5 , the old value is region=TestAcidGuarantees,,1732486521610.94f3ccf9f8a638fa974e1f69627832ef., hostname=bd6592c37bb2,40087,1732486423713, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94f3ccf9f8a638fa974e1f69627832ef, server=bd6592c37bb2,40087,1732486423713 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T22:15:29,413 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732486521610.94f3ccf9f8a638fa974e1f69627832ef., hostname=bd6592c37bb2,40087,1732486423713, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94f3ccf9f8a638fa974e1f69627832ef, server=bd6592c37bb2,40087,1732486423713 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T22:15:29,413 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40087 {}] ipc.CallRunner(138): callId: 36 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:32774 deadline: 1732486589401, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94f3ccf9f8a638fa974e1f69627832ef, server=bd6592c37bb2,40087,1732486423713 2024-11-24T22:15:29,413 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732486521610.94f3ccf9f8a638fa974e1f69627832ef., hostname=bd6592c37bb2,40087,1732486423713, seqNum=5 because the exception is null or not the one we care about 2024-11-24T22:15:29,413 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40087 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94f3ccf9f8a638fa974e1f69627832ef, server=bd6592c37bb2,40087,1732486423713 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-24T22:15:29,413 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732486521610.94f3ccf9f8a638fa974e1f69627832ef., hostname=bd6592c37bb2,40087,1732486423713, seqNum=5 , the old value is region=TestAcidGuarantees,,1732486521610.94f3ccf9f8a638fa974e1f69627832ef., hostname=bd6592c37bb2,40087,1732486423713, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94f3ccf9f8a638fa974e1f69627832ef, server=bd6592c37bb2,40087,1732486423713 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T22:15:29,413 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732486521610.94f3ccf9f8a638fa974e1f69627832ef., hostname=bd6592c37bb2,40087,1732486423713, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94f3ccf9f8a638fa974e1f69627832ef, server=bd6592c37bb2,40087,1732486423713 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T22:15:29,413 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40087 {}] ipc.CallRunner(138): callId: 32 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:32846 deadline: 1732486589401, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94f3ccf9f8a638fa974e1f69627832ef, server=bd6592c37bb2,40087,1732486423713 2024-11-24T22:15:29,413 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732486521610.94f3ccf9f8a638fa974e1f69627832ef., hostname=bd6592c37bb2,40087,1732486423713, seqNum=5 because the exception is null or not the one we care about 2024-11-24T22:15:29,414 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732486521610.94f3ccf9f8a638fa974e1f69627832ef., hostname=bd6592c37bb2,40087,1732486423713, seqNum=5 , the old value is region=TestAcidGuarantees,,1732486521610.94f3ccf9f8a638fa974e1f69627832ef., hostname=bd6592c37bb2,40087,1732486423713, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94f3ccf9f8a638fa974e1f69627832ef, server=bd6592c37bb2,40087,1732486423713 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T22:15:29,414 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732486521610.94f3ccf9f8a638fa974e1f69627832ef., hostname=bd6592c37bb2,40087,1732486423713, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94f3ccf9f8a638fa974e1f69627832ef, server=bd6592c37bb2,40087,1732486423713 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T22:15:29,414 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732486521610.94f3ccf9f8a638fa974e1f69627832ef., hostname=bd6592c37bb2,40087,1732486423713, seqNum=5 because the exception is null or not the one we care about 2024-11-24T22:15:29,428 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T22:15:29,432 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241124cc8a4013fb4f4d0780b44c16d7bd24b5_94f3ccf9f8a638fa974e1f69627832ef to hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241124cc8a4013fb4f4d0780b44c16d7bd24b5_94f3ccf9f8a638fa974e1f69627832ef 2024-11-24T22:15:29,432 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/94f3ccf9f8a638fa974e1f69627832ef/.tmp/A/89ef506c9e084dff96cbbd975d3a86cb, store: [table=TestAcidGuarantees family=A region=94f3ccf9f8a638fa974e1f69627832ef] 2024-11-24T22:15:29,433 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/94f3ccf9f8a638fa974e1f69627832ef/.tmp/A/89ef506c9e084dff96cbbd975d3a86cb is 175, key is test_row_0/A:col10/1732486528887/Put/seqid=0 2024-11-24T22:15:29,433 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40087 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94f3ccf9f8a638fa974e1f69627832ef, server=bd6592c37bb2,40087,1732486423713 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-24T22:15:29,433 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40087 {}] ipc.CallRunner(138): callId: 34 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:32822 deadline: 1732486589420, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94f3ccf9f8a638fa974e1f69627832ef, server=bd6592c37bb2,40087,1732486423713 2024-11-24T22:15:29,433 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732486521610.94f3ccf9f8a638fa974e1f69627832ef., hostname=bd6592c37bb2,40087,1732486423713, seqNum=5 , the old value is region=TestAcidGuarantees,,1732486521610.94f3ccf9f8a638fa974e1f69627832ef., hostname=bd6592c37bb2,40087,1732486423713, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94f3ccf9f8a638fa974e1f69627832ef, server=bd6592c37bb2,40087,1732486423713 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T22:15:29,434 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732486521610.94f3ccf9f8a638fa974e1f69627832ef., hostname=bd6592c37bb2,40087,1732486423713, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94f3ccf9f8a638fa974e1f69627832ef, server=bd6592c37bb2,40087,1732486423713 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T22:15:29,434 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732486521610.94f3ccf9f8a638fa974e1f69627832ef., hostname=bd6592c37bb2,40087,1732486423713, seqNum=5 because the exception is null or not the one we care about 2024-11-24T22:15:29,438 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45929 is added to blk_1073742198_1374 (size=39549) 2024-11-24T22:15:29,475 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=40087 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=100 2024-11-24T22:15:29,476 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bd6592c37bb2:0-1 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732486521610.94f3ccf9f8a638fa974e1f69627832ef. 2024-11-24T22:15:29,476 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bd6592c37bb2:0-1 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732486521610.94f3ccf9f8a638fa974e1f69627832ef. as already flushing 2024-11-24T22:15:29,476 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bd6592c37bb2:0-1 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732486521610.94f3ccf9f8a638fa974e1f69627832ef. 2024-11-24T22:15:29,476 ERROR [RS_FLUSH_OPERATIONS-regionserver/bd6592c37bb2:0-1 {event_type=RS_FLUSH_REGIONS, pid=100}] handler.RSProcedureHandler(58): pid=100 java.io.IOException: Unable to complete flush {ENCODED => 94f3ccf9f8a638fa974e1f69627832ef, NAME => 'TestAcidGuarantees,,1732486521610.94f3ccf9f8a638fa974e1f69627832ef.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T22:15:29,476 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bd6592c37bb2:0-1 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=100 java.io.IOException: Unable to complete flush {ENCODED => 94f3ccf9f8a638fa974e1f69627832ef, NAME => 'TestAcidGuarantees,,1732486521610.94f3ccf9f8a638fa974e1f69627832ef.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T22:15:29,476 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37701 {}] master.HMaster(4177): Remote procedure failed, pid=100 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 94f3ccf9f8a638fa974e1f69627832ef, NAME => 'TestAcidGuarantees,,1732486521610.94f3ccf9f8a638fa974e1f69627832ef.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 94f3ccf9f8a638fa974e1f69627832ef, NAME => 'TestAcidGuarantees,,1732486521610.94f3ccf9f8a638fa974e1f69627832ef.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T22:15:29,491 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37701 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=99 2024-11-24T22:15:29,494 DEBUG [RS:0;bd6592c37bb2:40087-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/94f3ccf9f8a638fa974e1f69627832ef/.tmp/A/1b69565af4094bd79b4d460cb7671894 as hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/94f3ccf9f8a638fa974e1f69627832ef/A/1b69565af4094bd79b4d460cb7671894 2024-11-24T22:15:29,498 INFO [RS:0;bd6592c37bb2:40087-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 94f3ccf9f8a638fa974e1f69627832ef/A of 94f3ccf9f8a638fa974e1f69627832ef into 1b69565af4094bd79b4d460cb7671894(size=30.3 K), total size for store is 69.0 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-24T22:15:29,498 DEBUG [RS:0;bd6592c37bb2:40087-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 94f3ccf9f8a638fa974e1f69627832ef: 2024-11-24T22:15:29,498 INFO [RS:0;bd6592c37bb2:40087-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732486521610.94f3ccf9f8a638fa974e1f69627832ef., storeName=94f3ccf9f8a638fa974e1f69627832ef/A, priority=13, startTime=1732486528664; duration=0sec 2024-11-24T22:15:29,498 DEBUG [RS:0;bd6592c37bb2:40087-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=2), splitQueue=0 2024-11-24T22:15:29,498 DEBUG [RS:0;bd6592c37bb2:40087-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 94f3ccf9f8a638fa974e1f69627832ef:A 2024-11-24T22:15:29,498 DEBUG [RS:0;bd6592c37bb2:40087-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 94f3ccf9f8a638fa974e1f69627832ef:B 2024-11-24T22:15:29,498 DEBUG [RS:0;bd6592c37bb2:40087-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 4 compacting, 0 eligible, 16 blocking 2024-11-24T22:15:29,498 DEBUG [RS:0;bd6592c37bb2:40087-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 0 files of size 0 starting at candidate #-1 after considering 0 permutations with 0 in ratio 2024-11-24T22:15:29,498 DEBUG [RS:0;bd6592c37bb2:40087-shortCompactions-0 {}] compactions.SortedCompactionPolicy(232): Not compacting files because we only have 0 files ready for compaction. Need 3 to initiate. 2024-11-24T22:15:29,498 DEBUG [RS:0;bd6592c37bb2:40087-shortCompactions-0 {}] regionserver.CompactSplit(450): Not compacting TestAcidGuarantees,,1732486521610.94f3ccf9f8a638fa974e1f69627832ef. because compaction request was cancelled 2024-11-24T22:15:29,498 DEBUG [RS:0;bd6592c37bb2:40087-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 94f3ccf9f8a638fa974e1f69627832ef:C 2024-11-24T22:15:29,498 DEBUG [RS:0;bd6592c37bb2:40087-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 2 store files, 0 compacting, 2 eligible, 16 blocking 2024-11-24T22:15:29,499 DEBUG [RS:0;bd6592c37bb2:40087-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 0 files of size 0 starting at candidate #-1 after considering 0 permutations with 0 in ratio 2024-11-24T22:15:29,499 DEBUG [RS:0;bd6592c37bb2:40087-shortCompactions-0 {}] compactions.SortedCompactionPolicy(232): Not compacting files because we only have 0 files ready for compaction. Need 3 to initiate. 2024-11-24T22:15:29,499 DEBUG [RS:0;bd6592c37bb2:40087-shortCompactions-0 {}] regionserver.CompactSplit(450): Not compacting TestAcidGuarantees,,1732486521610.94f3ccf9f8a638fa974e1f69627832ef. because compaction request was cancelled 2024-11-24T22:15:29,499 DEBUG [RS:0;bd6592c37bb2:40087-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 94f3ccf9f8a638fa974e1f69627832ef:B 2024-11-24T22:15:29,516 DEBUG [RS:0;bd6592c37bb2:40087-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/94f3ccf9f8a638fa974e1f69627832ef/.tmp/C/267a2d67eb91491c826a9d1801fe9d29 as hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/94f3ccf9f8a638fa974e1f69627832ef/C/267a2d67eb91491c826a9d1801fe9d29 2024-11-24T22:15:29,520 INFO [RS:0;bd6592c37bb2:40087-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 94f3ccf9f8a638fa974e1f69627832ef/C of 94f3ccf9f8a638fa974e1f69627832ef into 267a2d67eb91491c826a9d1801fe9d29(size=11.9 K), total size for store is 11.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-24T22:15:29,521 DEBUG [RS:0;bd6592c37bb2:40087-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 94f3ccf9f8a638fa974e1f69627832ef: 2024-11-24T22:15:29,521 INFO [RS:0;bd6592c37bb2:40087-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732486521610.94f3ccf9f8a638fa974e1f69627832ef., storeName=94f3ccf9f8a638fa974e1f69627832ef/C, priority=12, startTime=1732486528944; duration=0sec 2024-11-24T22:15:29,521 DEBUG [RS:0;bd6592c37bb2:40087-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-24T22:15:29,521 DEBUG [RS:0;bd6592c37bb2:40087-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 94f3ccf9f8a638fa974e1f69627832ef:C 2024-11-24T22:15:29,628 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=40087 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=100 2024-11-24T22:15:29,629 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bd6592c37bb2:0-2 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732486521610.94f3ccf9f8a638fa974e1f69627832ef. 2024-11-24T22:15:29,629 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bd6592c37bb2:0-2 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732486521610.94f3ccf9f8a638fa974e1f69627832ef. as already flushing 2024-11-24T22:15:29,629 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bd6592c37bb2:0-2 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732486521610.94f3ccf9f8a638fa974e1f69627832ef. 2024-11-24T22:15:29,629 ERROR [RS_FLUSH_OPERATIONS-regionserver/bd6592c37bb2:0-2 {event_type=RS_FLUSH_REGIONS, pid=100}] handler.RSProcedureHandler(58): pid=100 java.io.IOException: Unable to complete flush {ENCODED => 94f3ccf9f8a638fa974e1f69627832ef, NAME => 'TestAcidGuarantees,,1732486521610.94f3ccf9f8a638fa974e1f69627832ef.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T22:15:29,629 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bd6592c37bb2:0-2 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=100 java.io.IOException: Unable to complete flush {ENCODED => 94f3ccf9f8a638fa974e1f69627832ef, NAME => 'TestAcidGuarantees,,1732486521610.94f3ccf9f8a638fa974e1f69627832ef.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T22:15:29,629 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37701 {}] master.HMaster(4177): Remote procedure failed, pid=100 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 94f3ccf9f8a638fa974e1f69627832ef, NAME => 'TestAcidGuarantees,,1732486521610.94f3ccf9f8a638fa974e1f69627832ef.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 94f3ccf9f8a638fa974e1f69627832ef, NAME => 'TestAcidGuarantees,,1732486521610.94f3ccf9f8a638fa974e1f69627832ef.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T22:15:29,722 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40087 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94f3ccf9f8a638fa974e1f69627832ef, server=bd6592c37bb2,40087,1732486423713 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-24T22:15:29,722 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40087 {}] ipc.CallRunner(138): callId: 34 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:32810 deadline: 1732486589710, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94f3ccf9f8a638fa974e1f69627832ef, server=bd6592c37bb2,40087,1732486423713 2024-11-24T22:15:29,722 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732486521610.94f3ccf9f8a638fa974e1f69627832ef., hostname=bd6592c37bb2,40087,1732486423713, seqNum=5 , the old value is region=TestAcidGuarantees,,1732486521610.94f3ccf9f8a638fa974e1f69627832ef., hostname=bd6592c37bb2,40087,1732486423713, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94f3ccf9f8a638fa974e1f69627832ef, server=bd6592c37bb2,40087,1732486423713 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T22:15:29,723 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732486521610.94f3ccf9f8a638fa974e1f69627832ef., hostname=bd6592c37bb2,40087,1732486423713, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94f3ccf9f8a638fa974e1f69627832ef, server=bd6592c37bb2,40087,1732486423713 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T22:15:29,723 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732486521610.94f3ccf9f8a638fa974e1f69627832ef., hostname=bd6592c37bb2,40087,1732486423713, seqNum=5 because the exception is null or not the one we care about 2024-11-24T22:15:29,732 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40087 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94f3ccf9f8a638fa974e1f69627832ef, server=bd6592c37bb2,40087,1732486423713 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-24T22:15:29,733 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40087 {}] ipc.CallRunner(138): callId: 18 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:32832 deadline: 1732486589721, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94f3ccf9f8a638fa974e1f69627832ef, server=bd6592c37bb2,40087,1732486423713 2024-11-24T22:15:29,733 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732486521610.94f3ccf9f8a638fa974e1f69627832ef., hostname=bd6592c37bb2,40087,1732486423713, seqNum=5 , the old value is region=TestAcidGuarantees,,1732486521610.94f3ccf9f8a638fa974e1f69627832ef., hostname=bd6592c37bb2,40087,1732486423713, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94f3ccf9f8a638fa974e1f69627832ef, server=bd6592c37bb2,40087,1732486423713 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T22:15:29,733 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732486521610.94f3ccf9f8a638fa974e1f69627832ef., hostname=bd6592c37bb2,40087,1732486423713, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94f3ccf9f8a638fa974e1f69627832ef, server=bd6592c37bb2,40087,1732486423713 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T22:15:29,733 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732486521610.94f3ccf9f8a638fa974e1f69627832ef., hostname=bd6592c37bb2,40087,1732486423713, seqNum=5 because the exception is null or not the one we care about 2024-11-24T22:15:29,733 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40087 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94f3ccf9f8a638fa974e1f69627832ef, server=bd6592c37bb2,40087,1732486423713 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-24T22:15:29,733 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40087 {}] ipc.CallRunner(138): callId: 37 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:32774 deadline: 1732486589721, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94f3ccf9f8a638fa974e1f69627832ef, server=bd6592c37bb2,40087,1732486423713 2024-11-24T22:15:29,733 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732486521610.94f3ccf9f8a638fa974e1f69627832ef., hostname=bd6592c37bb2,40087,1732486423713, seqNum=5 , the old value is region=TestAcidGuarantees,,1732486521610.94f3ccf9f8a638fa974e1f69627832ef., hostname=bd6592c37bb2,40087,1732486423713, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94f3ccf9f8a638fa974e1f69627832ef, server=bd6592c37bb2,40087,1732486423713 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T22:15:29,733 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40087 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94f3ccf9f8a638fa974e1f69627832ef, server=bd6592c37bb2,40087,1732486423713 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-24T22:15:29,733 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732486521610.94f3ccf9f8a638fa974e1f69627832ef., hostname=bd6592c37bb2,40087,1732486423713, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94f3ccf9f8a638fa974e1f69627832ef, server=bd6592c37bb2,40087,1732486423713 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T22:15:29,733 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732486521610.94f3ccf9f8a638fa974e1f69627832ef., hostname=bd6592c37bb2,40087,1732486423713, seqNum=5 because the exception is null or not the one we care about 2024-11-24T22:15:29,733 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40087 {}] ipc.CallRunner(138): callId: 33 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:32846 deadline: 1732486589721, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94f3ccf9f8a638fa974e1f69627832ef, server=bd6592c37bb2,40087,1732486423713 2024-11-24T22:15:29,734 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732486521610.94f3ccf9f8a638fa974e1f69627832ef., hostname=bd6592c37bb2,40087,1732486423713, seqNum=5 , the old value is region=TestAcidGuarantees,,1732486521610.94f3ccf9f8a638fa974e1f69627832ef., hostname=bd6592c37bb2,40087,1732486423713, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94f3ccf9f8a638fa974e1f69627832ef, server=bd6592c37bb2,40087,1732486423713 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T22:15:29,734 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732486521610.94f3ccf9f8a638fa974e1f69627832ef., hostname=bd6592c37bb2,40087,1732486423713, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94f3ccf9f8a638fa974e1f69627832ef, server=bd6592c37bb2,40087,1732486423713 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T22:15:29,734 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732486521610.94f3ccf9f8a638fa974e1f69627832ef., hostname=bd6592c37bb2,40087,1732486423713, seqNum=5 because the exception is null or not the one we care about 2024-11-24T22:15:29,750 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40087 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94f3ccf9f8a638fa974e1f69627832ef, server=bd6592c37bb2,40087,1732486423713 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-24T22:15:29,750 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40087 {}] ipc.CallRunner(138): callId: 35 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:32822 deadline: 1732486589740, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94f3ccf9f8a638fa974e1f69627832ef, server=bd6592c37bb2,40087,1732486423713 2024-11-24T22:15:29,751 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732486521610.94f3ccf9f8a638fa974e1f69627832ef., hostname=bd6592c37bb2,40087,1732486423713, seqNum=5 , the old value is region=TestAcidGuarantees,,1732486521610.94f3ccf9f8a638fa974e1f69627832ef., hostname=bd6592c37bb2,40087,1732486423713, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94f3ccf9f8a638fa974e1f69627832ef, server=bd6592c37bb2,40087,1732486423713 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T22:15:29,751 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732486521610.94f3ccf9f8a638fa974e1f69627832ef., hostname=bd6592c37bb2,40087,1732486423713, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94f3ccf9f8a638fa974e1f69627832ef, server=bd6592c37bb2,40087,1732486423713 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T22:15:29,751 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732486521610.94f3ccf9f8a638fa974e1f69627832ef., hostname=bd6592c37bb2,40087,1732486423713, seqNum=5 because the exception is null or not the one we care about 2024-11-24T22:15:29,781 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=40087 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=100 2024-11-24T22:15:29,781 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bd6592c37bb2:0-0 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732486521610.94f3ccf9f8a638fa974e1f69627832ef. 2024-11-24T22:15:29,781 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bd6592c37bb2:0-0 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732486521610.94f3ccf9f8a638fa974e1f69627832ef. as already flushing 2024-11-24T22:15:29,781 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bd6592c37bb2:0-0 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732486521610.94f3ccf9f8a638fa974e1f69627832ef. 2024-11-24T22:15:29,782 ERROR [RS_FLUSH_OPERATIONS-regionserver/bd6592c37bb2:0-0 {event_type=RS_FLUSH_REGIONS, pid=100}] handler.RSProcedureHandler(58): pid=100 java.io.IOException: Unable to complete flush {ENCODED => 94f3ccf9f8a638fa974e1f69627832ef, NAME => 'TestAcidGuarantees,,1732486521610.94f3ccf9f8a638fa974e1f69627832ef.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T22:15:29,782 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bd6592c37bb2:0-0 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=100 java.io.IOException: Unable to complete flush {ENCODED => 94f3ccf9f8a638fa974e1f69627832ef, NAME => 'TestAcidGuarantees,,1732486521610.94f3ccf9f8a638fa974e1f69627832ef.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T22:15:29,782 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37701 {}] master.HMaster(4177): Remote procedure failed, pid=100 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 94f3ccf9f8a638fa974e1f69627832ef, NAME => 'TestAcidGuarantees,,1732486521610.94f3ccf9f8a638fa974e1f69627832ef.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 94f3ccf9f8a638fa974e1f69627832ef, NAME => 'TestAcidGuarantees,,1732486521610.94f3ccf9f8a638fa974e1f69627832ef.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T22:15:29,837 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=82, memsize=22.4 K, hasBloomFilter=true, into tmp file hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/94f3ccf9f8a638fa974e1f69627832ef/.tmp/A/89ef506c9e084dff96cbbd975d3a86cb 2024-11-24T22:15:29,844 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/94f3ccf9f8a638fa974e1f69627832ef/.tmp/B/f12630c898be400d87e378aa2a6ebac3 is 50, key is test_row_0/B:col10/1732486528887/Put/seqid=0 2024-11-24T22:15:29,848 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45929 is added to blk_1073742199_1375 (size=12001) 2024-11-24T22:15:29,849 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=82 (bloomFilter=true), to=hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/94f3ccf9f8a638fa974e1f69627832ef/.tmp/B/f12630c898be400d87e378aa2a6ebac3 2024-11-24T22:15:29,855 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/94f3ccf9f8a638fa974e1f69627832ef/.tmp/C/9b339451e3df4507bf90584aa37d34bd is 50, key is test_row_0/C:col10/1732486528887/Put/seqid=0 2024-11-24T22:15:29,861 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45929 is added to blk_1073742200_1376 (size=12001) 2024-11-24T22:15:29,861 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=82 (bloomFilter=true), to=hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/94f3ccf9f8a638fa974e1f69627832ef/.tmp/C/9b339451e3df4507bf90584aa37d34bd 2024-11-24T22:15:29,865 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/94f3ccf9f8a638fa974e1f69627832ef/.tmp/A/89ef506c9e084dff96cbbd975d3a86cb as hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/94f3ccf9f8a638fa974e1f69627832ef/A/89ef506c9e084dff96cbbd975d3a86cb 2024-11-24T22:15:29,868 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/94f3ccf9f8a638fa974e1f69627832ef/A/89ef506c9e084dff96cbbd975d3a86cb, entries=200, sequenceid=82, filesize=38.6 K 2024-11-24T22:15:29,869 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/94f3ccf9f8a638fa974e1f69627832ef/.tmp/B/f12630c898be400d87e378aa2a6ebac3 as hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/94f3ccf9f8a638fa974e1f69627832ef/B/f12630c898be400d87e378aa2a6ebac3 2024-11-24T22:15:29,872 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/94f3ccf9f8a638fa974e1f69627832ef/B/f12630c898be400d87e378aa2a6ebac3, entries=150, sequenceid=82, filesize=11.7 K 2024-11-24T22:15:29,872 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/94f3ccf9f8a638fa974e1f69627832ef/.tmp/C/9b339451e3df4507bf90584aa37d34bd as hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/94f3ccf9f8a638fa974e1f69627832ef/C/9b339451e3df4507bf90584aa37d34bd 2024-11-24T22:15:29,875 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/94f3ccf9f8a638fa974e1f69627832ef/C/9b339451e3df4507bf90584aa37d34bd, entries=150, sequenceid=82, filesize=11.7 K 2024-11-24T22:15:29,876 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~67.09 KB/68700, heapSize ~199.92 KB/204720, currentSize=107.34 KB/109920 for 94f3ccf9f8a638fa974e1f69627832ef in 861ms, sequenceid=82, compaction requested=true 2024-11-24T22:15:29,876 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 94f3ccf9f8a638fa974e1f69627832ef: 2024-11-24T22:15:29,876 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 94f3ccf9f8a638fa974e1f69627832ef:A, priority=-2147483648, current under compaction store size is 1 2024-11-24T22:15:29,876 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-24T22:15:29,876 DEBUG [RS:0;bd6592c37bb2:40087-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-24T22:15:29,876 DEBUG [RS:0;bd6592c37bb2:40087-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-24T22:15:29,876 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 94f3ccf9f8a638fa974e1f69627832ef:B, priority=-2147483648, current under compaction store size is 2 2024-11-24T22:15:29,876 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-24T22:15:29,876 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 94f3ccf9f8a638fa974e1f69627832ef:C, priority=-2147483648, current under compaction store size is 3 2024-11-24T22:15:29,876 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-24T22:15:29,879 DEBUG [RS:0;bd6592c37bb2:40087-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36106 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-24T22:15:29,879 DEBUG [RS:0;bd6592c37bb2:40087-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 110156 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-24T22:15:29,879 DEBUG [RS:0;bd6592c37bb2:40087-longCompactions-0 {}] regionserver.HStore(1541): 94f3ccf9f8a638fa974e1f69627832ef/B is initiating minor compaction (all files) 2024-11-24T22:15:29,879 DEBUG [RS:0;bd6592c37bb2:40087-shortCompactions-0 {}] regionserver.HStore(1541): 94f3ccf9f8a638fa974e1f69627832ef/A is initiating minor compaction (all files) 2024-11-24T22:15:29,879 INFO [RS:0;bd6592c37bb2:40087-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 94f3ccf9f8a638fa974e1f69627832ef/B in TestAcidGuarantees,,1732486521610.94f3ccf9f8a638fa974e1f69627832ef. 2024-11-24T22:15:29,879 INFO [RS:0;bd6592c37bb2:40087-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 94f3ccf9f8a638fa974e1f69627832ef/A in TestAcidGuarantees,,1732486521610.94f3ccf9f8a638fa974e1f69627832ef. 2024-11-24T22:15:29,879 INFO [RS:0;bd6592c37bb2:40087-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/94f3ccf9f8a638fa974e1f69627832ef/B/e6bb3a30f1f848dd9dea06828499b452, hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/94f3ccf9f8a638fa974e1f69627832ef/B/26545f645a554d1daddca6dbf2adc469, hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/94f3ccf9f8a638fa974e1f69627832ef/B/f12630c898be400d87e378aa2a6ebac3] into tmpdir=hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/94f3ccf9f8a638fa974e1f69627832ef/.tmp, totalSize=35.3 K 2024-11-24T22:15:29,879 INFO [RS:0;bd6592c37bb2:40087-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/94f3ccf9f8a638fa974e1f69627832ef/A/1b69565af4094bd79b4d460cb7671894, hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/94f3ccf9f8a638fa974e1f69627832ef/A/95ed96d635f54bce8685e4aa528975e7, hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/94f3ccf9f8a638fa974e1f69627832ef/A/89ef506c9e084dff96cbbd975d3a86cb] into tmpdir=hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/94f3ccf9f8a638fa974e1f69627832ef/.tmp, totalSize=107.6 K 2024-11-24T22:15:29,879 INFO [RS:0;bd6592c37bb2:40087-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=13 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1732486521610.94f3ccf9f8a638fa974e1f69627832ef. 2024-11-24T22:15:29,879 DEBUG [RS:0;bd6592c37bb2:40087-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1732486521610.94f3ccf9f8a638fa974e1f69627832ef. files: [hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/94f3ccf9f8a638fa974e1f69627832ef/A/1b69565af4094bd79b4d460cb7671894, hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/94f3ccf9f8a638fa974e1f69627832ef/A/95ed96d635f54bce8685e4aa528975e7, hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/94f3ccf9f8a638fa974e1f69627832ef/A/89ef506c9e084dff96cbbd975d3a86cb] 2024-11-24T22:15:29,879 DEBUG [RS:0;bd6592c37bb2:40087-longCompactions-0 {}] compactions.Compactor(225): Compacting e6bb3a30f1f848dd9dea06828499b452, keycount=150, bloomtype=ROW, size=11.8 K, encoding=NONE, compression=NONE, seqNum=49, earliestPutTs=1732486526999 2024-11-24T22:15:29,880 DEBUG [RS:0;bd6592c37bb2:40087-shortCompactions-0 {}] compactions.Compactor(225): Compacting 1b69565af4094bd79b4d460cb7671894, keycount=150, bloomtype=ROW, size=30.3 K, encoding=NONE, compression=NONE, seqNum=49, earliestPutTs=1732486526999 2024-11-24T22:15:29,880 DEBUG [RS:0;bd6592c37bb2:40087-longCompactions-0 {}] compactions.Compactor(225): Compacting 26545f645a554d1daddca6dbf2adc469, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=69, earliestPutTs=1732486528203 2024-11-24T22:15:29,880 DEBUG [RS:0;bd6592c37bb2:40087-longCompactions-0 {}] compactions.Compactor(225): Compacting f12630c898be400d87e378aa2a6ebac3, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=82, earliestPutTs=1732486528887 2024-11-24T22:15:29,880 DEBUG [RS:0;bd6592c37bb2:40087-shortCompactions-0 {}] compactions.Compactor(225): Compacting 95ed96d635f54bce8685e4aa528975e7, keycount=200, bloomtype=ROW, size=38.6 K, encoding=NONE, compression=NONE, seqNum=69, earliestPutTs=1732486528203 2024-11-24T22:15:29,880 DEBUG [RS:0;bd6592c37bb2:40087-shortCompactions-0 {}] compactions.Compactor(225): Compacting 89ef506c9e084dff96cbbd975d3a86cb, keycount=200, bloomtype=ROW, size=38.6 K, encoding=NONE, compression=NONE, seqNum=82, earliestPutTs=1732486528887 2024-11-24T22:15:29,889 INFO [RS:0;bd6592c37bb2:40087-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=94f3ccf9f8a638fa974e1f69627832ef] 2024-11-24T22:15:29,890 DEBUG [RS:0;bd6592c37bb2:40087-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e2024112432be9729eaac460b8ecfcab0367309ec_94f3ccf9f8a638fa974e1f69627832ef store=[table=TestAcidGuarantees family=A region=94f3ccf9f8a638fa974e1f69627832ef] 2024-11-24T22:15:29,891 INFO [RS:0;bd6592c37bb2:40087-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 94f3ccf9f8a638fa974e1f69627832ef#B#compaction#319 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-24T22:15:29,891 DEBUG [RS:0;bd6592c37bb2:40087-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/94f3ccf9f8a638fa974e1f69627832ef/.tmp/B/d8beb157656c45efade39f49eb755238 is 50, key is test_row_0/B:col10/1732486528887/Put/seqid=0 2024-11-24T22:15:29,893 DEBUG [RS:0;bd6592c37bb2:40087-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e2024112432be9729eaac460b8ecfcab0367309ec_94f3ccf9f8a638fa974e1f69627832ef, store=[table=TestAcidGuarantees family=A region=94f3ccf9f8a638fa974e1f69627832ef] 2024-11-24T22:15:29,893 DEBUG [RS:0;bd6592c37bb2:40087-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024112432be9729eaac460b8ecfcab0367309ec_94f3ccf9f8a638fa974e1f69627832ef because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=94f3ccf9f8a638fa974e1f69627832ef] 2024-11-24T22:15:29,902 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45929 is added to blk_1073742201_1377 (size=12207) 2024-11-24T22:15:29,902 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45929 is added to blk_1073742202_1378 (size=4469) 2024-11-24T22:15:29,934 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=40087 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=100 2024-11-24T22:15:29,934 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bd6592c37bb2:0-1 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732486521610.94f3ccf9f8a638fa974e1f69627832ef. 2024-11-24T22:15:29,935 INFO [RS_FLUSH_OPERATIONS-regionserver/bd6592c37bb2:0-1 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.HRegion(2902): Flushing 94f3ccf9f8a638fa974e1f69627832ef 3/3 column families, dataSize=107.34 KB heapSize=319.50 KB 2024-11-24T22:15:29,935 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bd6592c37bb2:0-1 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 94f3ccf9f8a638fa974e1f69627832ef, store=A 2024-11-24T22:15:29,935 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bd6592c37bb2:0-1 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-24T22:15:29,935 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bd6592c37bb2:0-1 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 94f3ccf9f8a638fa974e1f69627832ef, store=B 2024-11-24T22:15:29,935 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bd6592c37bb2:0-1 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-24T22:15:29,935 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bd6592c37bb2:0-1 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 94f3ccf9f8a638fa974e1f69627832ef, store=C 2024-11-24T22:15:29,935 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bd6592c37bb2:0-1 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-24T22:15:29,942 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bd6592c37bb2:0-1 {event_type=RS_FLUSH_REGIONS, pid=100}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024112412ed540505d4421b925a2bae62d50a4e_94f3ccf9f8a638fa974e1f69627832ef is 50, key is test_row_0/A:col10/1732486529052/Put/seqid=0 2024-11-24T22:15:29,949 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45929 is added to blk_1073742203_1379 (size=12154) 2024-11-24T22:15:30,000 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37701 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=99 2024-11-24T22:15:30,236 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40087 {}] regionserver.HRegion(8855): Flush requested on 94f3ccf9f8a638fa974e1f69627832ef 2024-11-24T22:15:30,236 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732486521610.94f3ccf9f8a638fa974e1f69627832ef. as already flushing 2024-11-24T22:15:30,267 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40087 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94f3ccf9f8a638fa974e1f69627832ef, server=bd6592c37bb2,40087,1732486423713 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-24T22:15:30,267 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40087 {}] ipc.CallRunner(138): callId: 21 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:32832 deadline: 1732486590255, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94f3ccf9f8a638fa974e1f69627832ef, server=bd6592c37bb2,40087,1732486423713 2024-11-24T22:15:30,267 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732486521610.94f3ccf9f8a638fa974e1f69627832ef., hostname=bd6592c37bb2,40087,1732486423713, seqNum=5 , the old value is region=TestAcidGuarantees,,1732486521610.94f3ccf9f8a638fa974e1f69627832ef., hostname=bd6592c37bb2,40087,1732486423713, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94f3ccf9f8a638fa974e1f69627832ef, server=bd6592c37bb2,40087,1732486423713 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T22:15:30,267 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732486521610.94f3ccf9f8a638fa974e1f69627832ef., hostname=bd6592c37bb2,40087,1732486423713, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94f3ccf9f8a638fa974e1f69627832ef, server=bd6592c37bb2,40087,1732486423713 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T22:15:30,267 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732486521610.94f3ccf9f8a638fa974e1f69627832ef., hostname=bd6592c37bb2,40087,1732486423713, seqNum=5 because the exception is null or not the one we care about 2024-11-24T22:15:30,273 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40087 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94f3ccf9f8a638fa974e1f69627832ef, server=bd6592c37bb2,40087,1732486423713 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-24T22:15:30,273 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40087 {}] ipc.CallRunner(138): callId: 40 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:32774 deadline: 1732486590256, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94f3ccf9f8a638fa974e1f69627832ef, server=bd6592c37bb2,40087,1732486423713 2024-11-24T22:15:30,273 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732486521610.94f3ccf9f8a638fa974e1f69627832ef., hostname=bd6592c37bb2,40087,1732486423713, seqNum=5 , the old value is region=TestAcidGuarantees,,1732486521610.94f3ccf9f8a638fa974e1f69627832ef., hostname=bd6592c37bb2,40087,1732486423713, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94f3ccf9f8a638fa974e1f69627832ef, server=bd6592c37bb2,40087,1732486423713 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T22:15:30,273 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732486521610.94f3ccf9f8a638fa974e1f69627832ef., hostname=bd6592c37bb2,40087,1732486423713, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94f3ccf9f8a638fa974e1f69627832ef, server=bd6592c37bb2,40087,1732486423713 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T22:15:30,273 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732486521610.94f3ccf9f8a638fa974e1f69627832ef., hostname=bd6592c37bb2,40087,1732486423713, seqNum=5 because the exception is null or not the one we care about 2024-11-24T22:15:30,273 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40087 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94f3ccf9f8a638fa974e1f69627832ef, server=bd6592c37bb2,40087,1732486423713 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-24T22:15:30,274 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40087 {}] ipc.CallRunner(138): callId: 36 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:32822 deadline: 1732486590261, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94f3ccf9f8a638fa974e1f69627832ef, server=bd6592c37bb2,40087,1732486423713 2024-11-24T22:15:30,274 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732486521610.94f3ccf9f8a638fa974e1f69627832ef., hostname=bd6592c37bb2,40087,1732486423713, seqNum=5 , the old value is region=TestAcidGuarantees,,1732486521610.94f3ccf9f8a638fa974e1f69627832ef., hostname=bd6592c37bb2,40087,1732486423713, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94f3ccf9f8a638fa974e1f69627832ef, server=bd6592c37bb2,40087,1732486423713 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T22:15:30,274 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732486521610.94f3ccf9f8a638fa974e1f69627832ef., hostname=bd6592c37bb2,40087,1732486423713, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94f3ccf9f8a638fa974e1f69627832ef, server=bd6592c37bb2,40087,1732486423713 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T22:15:30,274 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732486521610.94f3ccf9f8a638fa974e1f69627832ef., hostname=bd6592c37bb2,40087,1732486423713, seqNum=5 because the exception is null or not the one we care about 2024-11-24T22:15:30,280 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40087 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94f3ccf9f8a638fa974e1f69627832ef, server=bd6592c37bb2,40087,1732486423713 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-24T22:15:30,280 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40087 {}] ipc.CallRunner(138): callId: 36 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:32846 deadline: 1732486590266, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94f3ccf9f8a638fa974e1f69627832ef, server=bd6592c37bb2,40087,1732486423713 2024-11-24T22:15:30,280 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732486521610.94f3ccf9f8a638fa974e1f69627832ef., hostname=bd6592c37bb2,40087,1732486423713, seqNum=5 , the old value is region=TestAcidGuarantees,,1732486521610.94f3ccf9f8a638fa974e1f69627832ef., hostname=bd6592c37bb2,40087,1732486423713, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94f3ccf9f8a638fa974e1f69627832ef, server=bd6592c37bb2,40087,1732486423713 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T22:15:30,280 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732486521610.94f3ccf9f8a638fa974e1f69627832ef., hostname=bd6592c37bb2,40087,1732486423713, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94f3ccf9f8a638fa974e1f69627832ef, server=bd6592c37bb2,40087,1732486423713 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T22:15:30,281 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732486521610.94f3ccf9f8a638fa974e1f69627832ef., hostname=bd6592c37bb2,40087,1732486423713, seqNum=5 because the exception is null or not the one we care about 2024-11-24T22:15:30,281 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40087 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94f3ccf9f8a638fa974e1f69627832ef, server=bd6592c37bb2,40087,1732486423713 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-24T22:15:30,281 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40087 {}] ipc.CallRunner(138): callId: 39 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:32810 deadline: 1732486590267, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94f3ccf9f8a638fa974e1f69627832ef, server=bd6592c37bb2,40087,1732486423713 2024-11-24T22:15:30,281 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732486521610.94f3ccf9f8a638fa974e1f69627832ef., hostname=bd6592c37bb2,40087,1732486423713, seqNum=5 , the old value is region=TestAcidGuarantees,,1732486521610.94f3ccf9f8a638fa974e1f69627832ef., hostname=bd6592c37bb2,40087,1732486423713, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94f3ccf9f8a638fa974e1f69627832ef, server=bd6592c37bb2,40087,1732486423713 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T22:15:30,281 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732486521610.94f3ccf9f8a638fa974e1f69627832ef., hostname=bd6592c37bb2,40087,1732486423713, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94f3ccf9f8a638fa974e1f69627832ef, server=bd6592c37bb2,40087,1732486423713 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T22:15:30,281 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732486521610.94f3ccf9f8a638fa974e1f69627832ef., hostname=bd6592c37bb2,40087,1732486423713, seqNum=5 because the exception is null or not the one we care about 2024-11-24T22:15:30,303 INFO [RS:0;bd6592c37bb2:40087-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 94f3ccf9f8a638fa974e1f69627832ef#A#compaction#318 average throughput is 0.06 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-24T22:15:30,304 DEBUG [RS:0;bd6592c37bb2:40087-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/94f3ccf9f8a638fa974e1f69627832ef/.tmp/A/83f546c8ac3a4c0dbfe435b59d1e9387 is 175, key is test_row_0/A:col10/1732486528887/Put/seqid=0 2024-11-24T22:15:30,308 DEBUG [RS:0;bd6592c37bb2:40087-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/94f3ccf9f8a638fa974e1f69627832ef/.tmp/B/d8beb157656c45efade39f49eb755238 as hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/94f3ccf9f8a638fa974e1f69627832ef/B/d8beb157656c45efade39f49eb755238 2024-11-24T22:15:30,309 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45929 is added to blk_1073742204_1380 (size=31161) 2024-11-24T22:15:30,319 INFO [RS:0;bd6592c37bb2:40087-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 94f3ccf9f8a638fa974e1f69627832ef/B of 94f3ccf9f8a638fa974e1f69627832ef into d8beb157656c45efade39f49eb755238(size=11.9 K), total size for store is 11.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-24T22:15:30,319 DEBUG [RS:0;bd6592c37bb2:40087-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 94f3ccf9f8a638fa974e1f69627832ef: 2024-11-24T22:15:30,319 INFO [RS:0;bd6592c37bb2:40087-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732486521610.94f3ccf9f8a638fa974e1f69627832ef., storeName=94f3ccf9f8a638fa974e1f69627832ef/B, priority=13, startTime=1732486529876; duration=0sec 2024-11-24T22:15:30,319 DEBUG [RS:0;bd6592c37bb2:40087-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-24T22:15:30,319 DEBUG [RS:0;bd6592c37bb2:40087-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 94f3ccf9f8a638fa974e1f69627832ef:B 2024-11-24T22:15:30,320 DEBUG [RS:0;bd6592c37bb2:40087-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 2 store files, 0 compacting, 2 eligible, 16 blocking 2024-11-24T22:15:30,320 DEBUG [RS:0;bd6592c37bb2:40087-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 0 files of size 0 starting at candidate #-1 after considering 0 permutations with 0 in ratio 2024-11-24T22:15:30,320 DEBUG [RS:0;bd6592c37bb2:40087-longCompactions-0 {}] compactions.SortedCompactionPolicy(232): Not compacting files because we only have 0 files ready for compaction. Need 3 to initiate. 2024-11-24T22:15:30,320 DEBUG [RS:0;bd6592c37bb2:40087-longCompactions-0 {}] regionserver.CompactSplit(450): Not compacting TestAcidGuarantees,,1732486521610.94f3ccf9f8a638fa974e1f69627832ef. because compaction request was cancelled 2024-11-24T22:15:30,320 DEBUG [RS:0;bd6592c37bb2:40087-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 94f3ccf9f8a638fa974e1f69627832ef:C 2024-11-24T22:15:30,322 DEBUG [RS:0;bd6592c37bb2:40087-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/94f3ccf9f8a638fa974e1f69627832ef/.tmp/A/83f546c8ac3a4c0dbfe435b59d1e9387 as hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/94f3ccf9f8a638fa974e1f69627832ef/A/83f546c8ac3a4c0dbfe435b59d1e9387 2024-11-24T22:15:30,338 INFO [RS:0;bd6592c37bb2:40087-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 94f3ccf9f8a638fa974e1f69627832ef/A of 94f3ccf9f8a638fa974e1f69627832ef into 83f546c8ac3a4c0dbfe435b59d1e9387(size=30.4 K), total size for store is 30.4 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-24T22:15:30,338 DEBUG [RS:0;bd6592c37bb2:40087-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 94f3ccf9f8a638fa974e1f69627832ef: 2024-11-24T22:15:30,338 INFO [RS:0;bd6592c37bb2:40087-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732486521610.94f3ccf9f8a638fa974e1f69627832ef., storeName=94f3ccf9f8a638fa974e1f69627832ef/A, priority=13, startTime=1732486529876; duration=0sec 2024-11-24T22:15:30,338 DEBUG [RS:0;bd6592c37bb2:40087-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-24T22:15:30,338 DEBUG [RS:0;bd6592c37bb2:40087-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 94f3ccf9f8a638fa974e1f69627832ef:A 2024-11-24T22:15:30,351 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bd6592c37bb2:0-1 {event_type=RS_FLUSH_REGIONS, pid=100}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T22:15:30,354 INFO [RS_FLUSH_OPERATIONS-regionserver/bd6592c37bb2:0-1 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024112412ed540505d4421b925a2bae62d50a4e_94f3ccf9f8a638fa974e1f69627832ef to hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024112412ed540505d4421b925a2bae62d50a4e_94f3ccf9f8a638fa974e1f69627832ef 2024-11-24T22:15:30,355 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bd6592c37bb2:0-1 {event_type=RS_FLUSH_REGIONS, pid=100}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/94f3ccf9f8a638fa974e1f69627832ef/.tmp/A/e78a5bea000d4b5088fbdfd1f6b3d2f8, store: [table=TestAcidGuarantees family=A region=94f3ccf9f8a638fa974e1f69627832ef] 2024-11-24T22:15:30,356 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bd6592c37bb2:0-1 {event_type=RS_FLUSH_REGIONS, pid=100}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/94f3ccf9f8a638fa974e1f69627832ef/.tmp/A/e78a5bea000d4b5088fbdfd1f6b3d2f8 is 175, key is test_row_0/A:col10/1732486529052/Put/seqid=0 2024-11-24T22:15:30,364 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45929 is added to blk_1073742205_1381 (size=30955) 2024-11-24T22:15:30,365 INFO [RS_FLUSH_OPERATIONS-regionserver/bd6592c37bb2:0-1 {event_type=RS_FLUSH_REGIONS, pid=100}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=104, memsize=35.8 K, hasBloomFilter=true, into tmp file hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/94f3ccf9f8a638fa974e1f69627832ef/.tmp/A/e78a5bea000d4b5088fbdfd1f6b3d2f8 2024-11-24T22:15:30,372 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bd6592c37bb2:0-1 {event_type=RS_FLUSH_REGIONS, pid=100}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/94f3ccf9f8a638fa974e1f69627832ef/.tmp/B/97bd3de9d98b49909095a4474b63e142 is 50, key is test_row_0/B:col10/1732486529052/Put/seqid=0 2024-11-24T22:15:30,379 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40087 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94f3ccf9f8a638fa974e1f69627832ef, server=bd6592c37bb2,40087,1732486423713 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-24T22:15:30,379 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40087 {}] ipc.CallRunner(138): callId: 22 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:32832 deadline: 1732486590372, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94f3ccf9f8a638fa974e1f69627832ef, server=bd6592c37bb2,40087,1732486423713 2024-11-24T22:15:30,380 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732486521610.94f3ccf9f8a638fa974e1f69627832ef., hostname=bd6592c37bb2,40087,1732486423713, seqNum=5 , the old value is region=TestAcidGuarantees,,1732486521610.94f3ccf9f8a638fa974e1f69627832ef., hostname=bd6592c37bb2,40087,1732486423713, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94f3ccf9f8a638fa974e1f69627832ef, server=bd6592c37bb2,40087,1732486423713 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T22:15:30,380 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732486521610.94f3ccf9f8a638fa974e1f69627832ef., hostname=bd6592c37bb2,40087,1732486423713, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94f3ccf9f8a638fa974e1f69627832ef, server=bd6592c37bb2,40087,1732486423713 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T22:15:30,380 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732486521610.94f3ccf9f8a638fa974e1f69627832ef., hostname=bd6592c37bb2,40087,1732486423713, seqNum=5 because the exception is null or not the one we care about 2024-11-24T22:15:30,383 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45929 is added to blk_1073742206_1382 (size=12001) 2024-11-24T22:15:30,392 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40087 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94f3ccf9f8a638fa974e1f69627832ef, server=bd6592c37bb2,40087,1732486423713 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-24T22:15:30,392 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40087 {}] ipc.CallRunner(138): callId: 41 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:32774 deadline: 1732486590381, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94f3ccf9f8a638fa974e1f69627832ef, server=bd6592c37bb2,40087,1732486423713 2024-11-24T22:15:30,392 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732486521610.94f3ccf9f8a638fa974e1f69627832ef., hostname=bd6592c37bb2,40087,1732486423713, seqNum=5 , the old value is region=TestAcidGuarantees,,1732486521610.94f3ccf9f8a638fa974e1f69627832ef., hostname=bd6592c37bb2,40087,1732486423713, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94f3ccf9f8a638fa974e1f69627832ef, server=bd6592c37bb2,40087,1732486423713 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T22:15:30,392 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732486521610.94f3ccf9f8a638fa974e1f69627832ef., hostname=bd6592c37bb2,40087,1732486423713, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94f3ccf9f8a638fa974e1f69627832ef, server=bd6592c37bb2,40087,1732486423713 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T22:15:30,392 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732486521610.94f3ccf9f8a638fa974e1f69627832ef., hostname=bd6592c37bb2,40087,1732486423713, seqNum=5 because the exception is null or not the one we care about 2024-11-24T22:15:30,394 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40087 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94f3ccf9f8a638fa974e1f69627832ef, server=bd6592c37bb2,40087,1732486423713 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-24T22:15:30,394 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40087 {}] ipc.CallRunner(138): callId: 37 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:32846 deadline: 1732486590390, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94f3ccf9f8a638fa974e1f69627832ef, server=bd6592c37bb2,40087,1732486423713 2024-11-24T22:15:30,394 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40087 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94f3ccf9f8a638fa974e1f69627832ef, server=bd6592c37bb2,40087,1732486423713 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-24T22:15:30,394 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732486521610.94f3ccf9f8a638fa974e1f69627832ef., hostname=bd6592c37bb2,40087,1732486423713, seqNum=5 , the old value is region=TestAcidGuarantees,,1732486521610.94f3ccf9f8a638fa974e1f69627832ef., hostname=bd6592c37bb2,40087,1732486423713, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94f3ccf9f8a638fa974e1f69627832ef, server=bd6592c37bb2,40087,1732486423713 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T22:15:30,394 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40087 {}] ipc.CallRunner(138): callId: 40 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:32810 deadline: 1732486590391, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94f3ccf9f8a638fa974e1f69627832ef, server=bd6592c37bb2,40087,1732486423713 2024-11-24T22:15:30,394 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732486521610.94f3ccf9f8a638fa974e1f69627832ef., hostname=bd6592c37bb2,40087,1732486423713, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94f3ccf9f8a638fa974e1f69627832ef, server=bd6592c37bb2,40087,1732486423713 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T22:15:30,394 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732486521610.94f3ccf9f8a638fa974e1f69627832ef., hostname=bd6592c37bb2,40087,1732486423713, seqNum=5 because the exception is null or not the one we care about 2024-11-24T22:15:30,395 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732486521610.94f3ccf9f8a638fa974e1f69627832ef., hostname=bd6592c37bb2,40087,1732486423713, seqNum=5 , the old value is region=TestAcidGuarantees,,1732486521610.94f3ccf9f8a638fa974e1f69627832ef., hostname=bd6592c37bb2,40087,1732486423713, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94f3ccf9f8a638fa974e1f69627832ef, server=bd6592c37bb2,40087,1732486423713 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T22:15:30,395 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732486521610.94f3ccf9f8a638fa974e1f69627832ef., hostname=bd6592c37bb2,40087,1732486423713, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94f3ccf9f8a638fa974e1f69627832ef, server=bd6592c37bb2,40087,1732486423713 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T22:15:30,395 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732486521610.94f3ccf9f8a638fa974e1f69627832ef., hostname=bd6592c37bb2,40087,1732486423713, seqNum=5 because the exception is null or not the one we care about 2024-11-24T22:15:30,599 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40087 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94f3ccf9f8a638fa974e1f69627832ef, server=bd6592c37bb2,40087,1732486423713 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-24T22:15:30,600 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40087 {}] ipc.CallRunner(138): callId: 23 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:32832 deadline: 1732486590591, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94f3ccf9f8a638fa974e1f69627832ef, server=bd6592c37bb2,40087,1732486423713 2024-11-24T22:15:30,600 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732486521610.94f3ccf9f8a638fa974e1f69627832ef., hostname=bd6592c37bb2,40087,1732486423713, seqNum=5 , the old value is region=TestAcidGuarantees,,1732486521610.94f3ccf9f8a638fa974e1f69627832ef., hostname=bd6592c37bb2,40087,1732486423713, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94f3ccf9f8a638fa974e1f69627832ef, server=bd6592c37bb2,40087,1732486423713 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T22:15:30,600 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732486521610.94f3ccf9f8a638fa974e1f69627832ef., hostname=bd6592c37bb2,40087,1732486423713, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94f3ccf9f8a638fa974e1f69627832ef, server=bd6592c37bb2,40087,1732486423713 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T22:15:30,600 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732486521610.94f3ccf9f8a638fa974e1f69627832ef., hostname=bd6592c37bb2,40087,1732486423713, seqNum=5 because the exception is null or not the one we care about 2024-11-24T22:15:30,610 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40087 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94f3ccf9f8a638fa974e1f69627832ef, server=bd6592c37bb2,40087,1732486423713 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-24T22:15:30,610 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40087 {}] ipc.CallRunner(138): callId: 42 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:32774 deadline: 1732486590600, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94f3ccf9f8a638fa974e1f69627832ef, server=bd6592c37bb2,40087,1732486423713 2024-11-24T22:15:30,610 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40087 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94f3ccf9f8a638fa974e1f69627832ef, server=bd6592c37bb2,40087,1732486423713 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-24T22:15:30,610 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732486521610.94f3ccf9f8a638fa974e1f69627832ef., hostname=bd6592c37bb2,40087,1732486423713, seqNum=5 , the old value is region=TestAcidGuarantees,,1732486521610.94f3ccf9f8a638fa974e1f69627832ef., hostname=bd6592c37bb2,40087,1732486423713, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94f3ccf9f8a638fa974e1f69627832ef, server=bd6592c37bb2,40087,1732486423713 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T22:15:30,610 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732486521610.94f3ccf9f8a638fa974e1f69627832ef., hostname=bd6592c37bb2,40087,1732486423713, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94f3ccf9f8a638fa974e1f69627832ef, server=bd6592c37bb2,40087,1732486423713 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T22:15:30,610 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40087 {}] ipc.CallRunner(138): callId: 38 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:32846 deadline: 1732486590600, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94f3ccf9f8a638fa974e1f69627832ef, server=bd6592c37bb2,40087,1732486423713 2024-11-24T22:15:30,610 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732486521610.94f3ccf9f8a638fa974e1f69627832ef., hostname=bd6592c37bb2,40087,1732486423713, seqNum=5 because the exception is null or not the one we care about 2024-11-24T22:15:30,610 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732486521610.94f3ccf9f8a638fa974e1f69627832ef., hostname=bd6592c37bb2,40087,1732486423713, seqNum=5 , the old value is region=TestAcidGuarantees,,1732486521610.94f3ccf9f8a638fa974e1f69627832ef., hostname=bd6592c37bb2,40087,1732486423713, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94f3ccf9f8a638fa974e1f69627832ef, server=bd6592c37bb2,40087,1732486423713 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T22:15:30,610 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40087 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94f3ccf9f8a638fa974e1f69627832ef, server=bd6592c37bb2,40087,1732486423713 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-24T22:15:30,610 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732486521610.94f3ccf9f8a638fa974e1f69627832ef., hostname=bd6592c37bb2,40087,1732486423713, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94f3ccf9f8a638fa974e1f69627832ef, server=bd6592c37bb2,40087,1732486423713 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T22:15:30,611 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732486521610.94f3ccf9f8a638fa974e1f69627832ef., hostname=bd6592c37bb2,40087,1732486423713, seqNum=5 because the exception is null or not the one we care about 2024-11-24T22:15:30,611 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40087 {}] ipc.CallRunner(138): callId: 41 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:32810 deadline: 1732486590601, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94f3ccf9f8a638fa974e1f69627832ef, server=bd6592c37bb2,40087,1732486423713 2024-11-24T22:15:30,611 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732486521610.94f3ccf9f8a638fa974e1f69627832ef., hostname=bd6592c37bb2,40087,1732486423713, seqNum=5 , the old value is region=TestAcidGuarantees,,1732486521610.94f3ccf9f8a638fa974e1f69627832ef., hostname=bd6592c37bb2,40087,1732486423713, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94f3ccf9f8a638fa974e1f69627832ef, server=bd6592c37bb2,40087,1732486423713 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T22:15:30,611 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732486521610.94f3ccf9f8a638fa974e1f69627832ef., hostname=bd6592c37bb2,40087,1732486423713, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94f3ccf9f8a638fa974e1f69627832ef, server=bd6592c37bb2,40087,1732486423713 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T22:15:30,611 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732486521610.94f3ccf9f8a638fa974e1f69627832ef., hostname=bd6592c37bb2,40087,1732486423713, seqNum=5 because the exception is null or not the one we care about 2024-11-24T22:15:30,784 INFO [RS_FLUSH_OPERATIONS-regionserver/bd6592c37bb2:0-1 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=35.78 KB at sequenceid=104 (bloomFilter=true), to=hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/94f3ccf9f8a638fa974e1f69627832ef/.tmp/B/97bd3de9d98b49909095a4474b63e142 2024-11-24T22:15:30,792 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bd6592c37bb2:0-1 {event_type=RS_FLUSH_REGIONS, pid=100}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/94f3ccf9f8a638fa974e1f69627832ef/.tmp/C/f90e75ec41424220862d0a6950395ea9 is 50, key is test_row_0/C:col10/1732486529052/Put/seqid=0 2024-11-24T22:15:30,799 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45929 is added to blk_1073742207_1383 (size=12001) 2024-11-24T22:15:30,917 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40087 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94f3ccf9f8a638fa974e1f69627832ef, server=bd6592c37bb2,40087,1732486423713 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-24T22:15:30,917 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40087 {}] ipc.CallRunner(138): callId: 24 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:32832 deadline: 1732486590911, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94f3ccf9f8a638fa974e1f69627832ef, server=bd6592c37bb2,40087,1732486423713 2024-11-24T22:15:30,918 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732486521610.94f3ccf9f8a638fa974e1f69627832ef., hostname=bd6592c37bb2,40087,1732486423713, seqNum=5 , the old value is region=TestAcidGuarantees,,1732486521610.94f3ccf9f8a638fa974e1f69627832ef., hostname=bd6592c37bb2,40087,1732486423713, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94f3ccf9f8a638fa974e1f69627832ef, server=bd6592c37bb2,40087,1732486423713 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T22:15:30,918 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732486521610.94f3ccf9f8a638fa974e1f69627832ef., hostname=bd6592c37bb2,40087,1732486423713, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94f3ccf9f8a638fa974e1f69627832ef, server=bd6592c37bb2,40087,1732486423713 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T22:15:30,918 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732486521610.94f3ccf9f8a638fa974e1f69627832ef., hostname=bd6592c37bb2,40087,1732486423713, seqNum=5 because the exception is null or not the one we care about 2024-11-24T22:15:30,932 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40087 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94f3ccf9f8a638fa974e1f69627832ef, server=bd6592c37bb2,40087,1732486423713 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-24T22:15:30,932 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40087 {}] ipc.CallRunner(138): callId: 43 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:32774 deadline: 1732486590921, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94f3ccf9f8a638fa974e1f69627832ef, server=bd6592c37bb2,40087,1732486423713 2024-11-24T22:15:30,932 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40087 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94f3ccf9f8a638fa974e1f69627832ef, server=bd6592c37bb2,40087,1732486423713 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-24T22:15:30,932 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40087 {}] ipc.CallRunner(138): callId: 39 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:32846 deadline: 1732486590921, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94f3ccf9f8a638fa974e1f69627832ef, server=bd6592c37bb2,40087,1732486423713 2024-11-24T22:15:30,932 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732486521610.94f3ccf9f8a638fa974e1f69627832ef., hostname=bd6592c37bb2,40087,1732486423713, seqNum=5 , the old value is region=TestAcidGuarantees,,1732486521610.94f3ccf9f8a638fa974e1f69627832ef., hostname=bd6592c37bb2,40087,1732486423713, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94f3ccf9f8a638fa974e1f69627832ef, server=bd6592c37bb2,40087,1732486423713 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T22:15:30,932 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732486521610.94f3ccf9f8a638fa974e1f69627832ef., hostname=bd6592c37bb2,40087,1732486423713, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94f3ccf9f8a638fa974e1f69627832ef, server=bd6592c37bb2,40087,1732486423713 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T22:15:30,932 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732486521610.94f3ccf9f8a638fa974e1f69627832ef., hostname=bd6592c37bb2,40087,1732486423713, seqNum=5 because the exception is null or not the one we care about 2024-11-24T22:15:30,932 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40087 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94f3ccf9f8a638fa974e1f69627832ef, server=bd6592c37bb2,40087,1732486423713 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-24T22:15:30,932 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732486521610.94f3ccf9f8a638fa974e1f69627832ef., hostname=bd6592c37bb2,40087,1732486423713, seqNum=5 , the old value is region=TestAcidGuarantees,,1732486521610.94f3ccf9f8a638fa974e1f69627832ef., hostname=bd6592c37bb2,40087,1732486423713, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94f3ccf9f8a638fa974e1f69627832ef, server=bd6592c37bb2,40087,1732486423713 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T22:15:30,933 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732486521610.94f3ccf9f8a638fa974e1f69627832ef., hostname=bd6592c37bb2,40087,1732486423713, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94f3ccf9f8a638fa974e1f69627832ef, server=bd6592c37bb2,40087,1732486423713 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T22:15:30,933 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40087 {}] ipc.CallRunner(138): callId: 42 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:32810 deadline: 1732486590921, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94f3ccf9f8a638fa974e1f69627832ef, server=bd6592c37bb2,40087,1732486423713 2024-11-24T22:15:30,933 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732486521610.94f3ccf9f8a638fa974e1f69627832ef., hostname=bd6592c37bb2,40087,1732486423713, seqNum=5 because the exception is null or not the one we care about 2024-11-24T22:15:30,933 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732486521610.94f3ccf9f8a638fa974e1f69627832ef., hostname=bd6592c37bb2,40087,1732486423713, seqNum=5 , the old value is region=TestAcidGuarantees,,1732486521610.94f3ccf9f8a638fa974e1f69627832ef., hostname=bd6592c37bb2,40087,1732486423713, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94f3ccf9f8a638fa974e1f69627832ef, server=bd6592c37bb2,40087,1732486423713 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T22:15:30,933 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732486521610.94f3ccf9f8a638fa974e1f69627832ef., hostname=bd6592c37bb2,40087,1732486423713, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94f3ccf9f8a638fa974e1f69627832ef, server=bd6592c37bb2,40087,1732486423713 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T22:15:30,933 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732486521610.94f3ccf9f8a638fa974e1f69627832ef., hostname=bd6592c37bb2,40087,1732486423713, seqNum=5 because the exception is null or not the one we care about 2024-11-24T22:15:31,011 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37701 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=99 2024-11-24T22:15:31,197 INFO [RS_FLUSH_OPERATIONS-regionserver/bd6592c37bb2:0-1 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=35.78 KB at sequenceid=104 (bloomFilter=true), to=hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/94f3ccf9f8a638fa974e1f69627832ef/.tmp/C/f90e75ec41424220862d0a6950395ea9 2024-11-24T22:15:31,201 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bd6592c37bb2:0-1 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/94f3ccf9f8a638fa974e1f69627832ef/.tmp/A/e78a5bea000d4b5088fbdfd1f6b3d2f8 as hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/94f3ccf9f8a638fa974e1f69627832ef/A/e78a5bea000d4b5088fbdfd1f6b3d2f8 2024-11-24T22:15:31,204 INFO [RS_FLUSH_OPERATIONS-regionserver/bd6592c37bb2:0-1 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/94f3ccf9f8a638fa974e1f69627832ef/A/e78a5bea000d4b5088fbdfd1f6b3d2f8, entries=150, sequenceid=104, filesize=30.2 K 2024-11-24T22:15:31,205 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bd6592c37bb2:0-1 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/94f3ccf9f8a638fa974e1f69627832ef/.tmp/B/97bd3de9d98b49909095a4474b63e142 as hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/94f3ccf9f8a638fa974e1f69627832ef/B/97bd3de9d98b49909095a4474b63e142 2024-11-24T22:15:31,208 INFO [RS_FLUSH_OPERATIONS-regionserver/bd6592c37bb2:0-1 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/94f3ccf9f8a638fa974e1f69627832ef/B/97bd3de9d98b49909095a4474b63e142, entries=150, sequenceid=104, filesize=11.7 K 2024-11-24T22:15:31,209 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bd6592c37bb2:0-1 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/94f3ccf9f8a638fa974e1f69627832ef/.tmp/C/f90e75ec41424220862d0a6950395ea9 as hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/94f3ccf9f8a638fa974e1f69627832ef/C/f90e75ec41424220862d0a6950395ea9 2024-11-24T22:15:31,212 INFO [RS_FLUSH_OPERATIONS-regionserver/bd6592c37bb2:0-1 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/94f3ccf9f8a638fa974e1f69627832ef/C/f90e75ec41424220862d0a6950395ea9, entries=150, sequenceid=104, filesize=11.7 K 2024-11-24T22:15:31,213 INFO [RS_FLUSH_OPERATIONS-regionserver/bd6592c37bb2:0-1 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.HRegion(3140): Finished flush of dataSize ~107.34 KB/109920, heapSize ~319.45 KB/327120, currentSize=67.09 KB/68700 for 94f3ccf9f8a638fa974e1f69627832ef in 1278ms, sequenceid=104, compaction requested=true 2024-11-24T22:15:31,213 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bd6592c37bb2:0-1 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.HRegion(2603): Flush status journal for 94f3ccf9f8a638fa974e1f69627832ef: 2024-11-24T22:15:31,213 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bd6592c37bb2:0-1 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732486521610.94f3ccf9f8a638fa974e1f69627832ef. 2024-11-24T22:15:31,213 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bd6592c37bb2:0-1 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=100 2024-11-24T22:15:31,214 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37701 {}] master.HMaster(4169): Remote procedure done, pid=100 2024-11-24T22:15:31,216 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=100, resume processing ppid=99 2024-11-24T22:15:31,216 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=100, ppid=99, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 2.3500 sec 2024-11-24T22:15:31,217 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=99, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=99, table=TestAcidGuarantees in 2.3540 sec 2024-11-24T22:15:31,288 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40087 {}] regionserver.HRegion(8855): Flush requested on 94f3ccf9f8a638fa974e1f69627832ef 2024-11-24T22:15:31,289 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 94f3ccf9f8a638fa974e1f69627832ef 3/3 column families, dataSize=73.80 KB heapSize=219.89 KB 2024-11-24T22:15:31,289 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 94f3ccf9f8a638fa974e1f69627832ef, store=A 2024-11-24T22:15:31,289 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-24T22:15:31,289 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 94f3ccf9f8a638fa974e1f69627832ef, store=B 2024-11-24T22:15:31,289 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-24T22:15:31,289 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 94f3ccf9f8a638fa974e1f69627832ef, store=C 2024-11-24T22:15:31,289 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-24T22:15:31,295 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241124250a21fc5d534c768d85cee87472e326_94f3ccf9f8a638fa974e1f69627832ef is 50, key is test_row_0/A:col10/1732486530255/Put/seqid=0 2024-11-24T22:15:31,310 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45929 is added to blk_1073742208_1384 (size=14594) 2024-11-24T22:15:31,430 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40087 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94f3ccf9f8a638fa974e1f69627832ef, server=bd6592c37bb2,40087,1732486423713 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-24T22:15:31,431 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40087 {}] ipc.CallRunner(138): callId: 53 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:32822 deadline: 1732486591417, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94f3ccf9f8a638fa974e1f69627832ef, server=bd6592c37bb2,40087,1732486423713 2024-11-24T22:15:31,431 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40087 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94f3ccf9f8a638fa974e1f69627832ef, server=bd6592c37bb2,40087,1732486423713 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-24T22:15:31,431 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732486521610.94f3ccf9f8a638fa974e1f69627832ef., hostname=bd6592c37bb2,40087,1732486423713, seqNum=5 , the old value is region=TestAcidGuarantees,,1732486521610.94f3ccf9f8a638fa974e1f69627832ef., hostname=bd6592c37bb2,40087,1732486423713, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94f3ccf9f8a638fa974e1f69627832ef, server=bd6592c37bb2,40087,1732486423713 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T22:15:31,431 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40087 {}] ipc.CallRunner(138): callId: 25 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:32832 deadline: 1732486591421, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94f3ccf9f8a638fa974e1f69627832ef, server=bd6592c37bb2,40087,1732486423713 2024-11-24T22:15:31,431 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732486521610.94f3ccf9f8a638fa974e1f69627832ef., hostname=bd6592c37bb2,40087,1732486423713, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94f3ccf9f8a638fa974e1f69627832ef, server=bd6592c37bb2,40087,1732486423713 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T22:15:31,431 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732486521610.94f3ccf9f8a638fa974e1f69627832ef., hostname=bd6592c37bb2,40087,1732486423713, seqNum=5 because the exception is null or not the one we care about 2024-11-24T22:15:31,431 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732486521610.94f3ccf9f8a638fa974e1f69627832ef., hostname=bd6592c37bb2,40087,1732486423713, seqNum=5 , the old value is region=TestAcidGuarantees,,1732486521610.94f3ccf9f8a638fa974e1f69627832ef., hostname=bd6592c37bb2,40087,1732486423713, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94f3ccf9f8a638fa974e1f69627832ef, server=bd6592c37bb2,40087,1732486423713 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T22:15:31,431 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732486521610.94f3ccf9f8a638fa974e1f69627832ef., hostname=bd6592c37bb2,40087,1732486423713, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94f3ccf9f8a638fa974e1f69627832ef, server=bd6592c37bb2,40087,1732486423713 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T22:15:31,432 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732486521610.94f3ccf9f8a638fa974e1f69627832ef., hostname=bd6592c37bb2,40087,1732486423713, seqNum=5 because the exception is null or not the one we care about 2024-11-24T22:15:31,447 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40087 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94f3ccf9f8a638fa974e1f69627832ef, server=bd6592c37bb2,40087,1732486423713 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-24T22:15:31,448 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40087 {}] ipc.CallRunner(138): callId: 44 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:32774 deadline: 1732486591440, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94f3ccf9f8a638fa974e1f69627832ef, server=bd6592c37bb2,40087,1732486423713 2024-11-24T22:15:31,448 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40087 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94f3ccf9f8a638fa974e1f69627832ef, server=bd6592c37bb2,40087,1732486423713 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-24T22:15:31,448 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732486521610.94f3ccf9f8a638fa974e1f69627832ef., hostname=bd6592c37bb2,40087,1732486423713, seqNum=5 , the old value is region=TestAcidGuarantees,,1732486521610.94f3ccf9f8a638fa974e1f69627832ef., hostname=bd6592c37bb2,40087,1732486423713, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94f3ccf9f8a638fa974e1f69627832ef, server=bd6592c37bb2,40087,1732486423713 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T22:15:31,448 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732486521610.94f3ccf9f8a638fa974e1f69627832ef., hostname=bd6592c37bb2,40087,1732486423713, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94f3ccf9f8a638fa974e1f69627832ef, server=bd6592c37bb2,40087,1732486423713 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T22:15:31,448 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40087 {}] ipc.CallRunner(138): callId: 40 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:32846 deadline: 1732486591440, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94f3ccf9f8a638fa974e1f69627832ef, server=bd6592c37bb2,40087,1732486423713 2024-11-24T22:15:31,448 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732486521610.94f3ccf9f8a638fa974e1f69627832ef., hostname=bd6592c37bb2,40087,1732486423713, seqNum=5 because the exception is null or not the one we care about 2024-11-24T22:15:31,448 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732486521610.94f3ccf9f8a638fa974e1f69627832ef., hostname=bd6592c37bb2,40087,1732486423713, seqNum=5 , the old value is region=TestAcidGuarantees,,1732486521610.94f3ccf9f8a638fa974e1f69627832ef., hostname=bd6592c37bb2,40087,1732486423713, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94f3ccf9f8a638fa974e1f69627832ef, server=bd6592c37bb2,40087,1732486423713 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T22:15:31,448 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732486521610.94f3ccf9f8a638fa974e1f69627832ef., hostname=bd6592c37bb2,40087,1732486423713, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94f3ccf9f8a638fa974e1f69627832ef, server=bd6592c37bb2,40087,1732486423713 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T22:15:31,448 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40087 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94f3ccf9f8a638fa974e1f69627832ef, server=bd6592c37bb2,40087,1732486423713 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-24T22:15:31,448 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732486521610.94f3ccf9f8a638fa974e1f69627832ef., hostname=bd6592c37bb2,40087,1732486423713, seqNum=5 because the exception is null or not the one we care about 2024-11-24T22:15:31,448 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40087 {}] ipc.CallRunner(138): callId: 43 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:32810 deadline: 1732486591440, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94f3ccf9f8a638fa974e1f69627832ef, server=bd6592c37bb2,40087,1732486423713 2024-11-24T22:15:31,449 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732486521610.94f3ccf9f8a638fa974e1f69627832ef., hostname=bd6592c37bb2,40087,1732486423713, seqNum=5 , the old value is region=TestAcidGuarantees,,1732486521610.94f3ccf9f8a638fa974e1f69627832ef., hostname=bd6592c37bb2,40087,1732486423713, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94f3ccf9f8a638fa974e1f69627832ef, server=bd6592c37bb2,40087,1732486423713 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T22:15:31,449 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732486521610.94f3ccf9f8a638fa974e1f69627832ef., hostname=bd6592c37bb2,40087,1732486423713, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94f3ccf9f8a638fa974e1f69627832ef, server=bd6592c37bb2,40087,1732486423713 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T22:15:31,449 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732486521610.94f3ccf9f8a638fa974e1f69627832ef., hostname=bd6592c37bb2,40087,1732486423713, seqNum=5 because the exception is null or not the one we care about 2024-11-24T22:15:31,549 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40087 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94f3ccf9f8a638fa974e1f69627832ef, server=bd6592c37bb2,40087,1732486423713 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-24T22:15:31,550 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40087 {}] ipc.CallRunner(138): callId: 54 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:32822 deadline: 1732486591540, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94f3ccf9f8a638fa974e1f69627832ef, server=bd6592c37bb2,40087,1732486423713 2024-11-24T22:15:31,550 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732486521610.94f3ccf9f8a638fa974e1f69627832ef., hostname=bd6592c37bb2,40087,1732486423713, seqNum=5 , the old value is region=TestAcidGuarantees,,1732486521610.94f3ccf9f8a638fa974e1f69627832ef., hostname=bd6592c37bb2,40087,1732486423713, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94f3ccf9f8a638fa974e1f69627832ef, server=bd6592c37bb2,40087,1732486423713 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T22:15:31,550 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732486521610.94f3ccf9f8a638fa974e1f69627832ef., hostname=bd6592c37bb2,40087,1732486423713, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94f3ccf9f8a638fa974e1f69627832ef, server=bd6592c37bb2,40087,1732486423713 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T22:15:31,550 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732486521610.94f3ccf9f8a638fa974e1f69627832ef., hostname=bd6592c37bb2,40087,1732486423713, seqNum=5 because the exception is null or not the one we care about 2024-11-24T22:15:31,711 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T22:15:31,717 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241124250a21fc5d534c768d85cee87472e326_94f3ccf9f8a638fa974e1f69627832ef to hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241124250a21fc5d534c768d85cee87472e326_94f3ccf9f8a638fa974e1f69627832ef 2024-11-24T22:15:31,718 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/94f3ccf9f8a638fa974e1f69627832ef/.tmp/A/3e23a020d2d641a09079ca5e307d947e, store: [table=TestAcidGuarantees family=A region=94f3ccf9f8a638fa974e1f69627832ef] 2024-11-24T22:15:31,719 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/94f3ccf9f8a638fa974e1f69627832ef/.tmp/A/3e23a020d2d641a09079ca5e307d947e is 175, key is test_row_0/A:col10/1732486530255/Put/seqid=0 2024-11-24T22:15:31,756 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45929 is added to blk_1073742209_1385 (size=39549) 2024-11-24T22:15:31,757 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=120, memsize=24.6 K, hasBloomFilter=true, into tmp file hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/94f3ccf9f8a638fa974e1f69627832ef/.tmp/A/3e23a020d2d641a09079ca5e307d947e 2024-11-24T22:15:31,769 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/94f3ccf9f8a638fa974e1f69627832ef/.tmp/B/1c3c63e6b7a943afbb4d603545e4dd0a is 50, key is test_row_0/B:col10/1732486530255/Put/seqid=0 2024-11-24T22:15:31,775 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40087 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94f3ccf9f8a638fa974e1f69627832ef, server=bd6592c37bb2,40087,1732486423713 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-24T22:15:31,775 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40087 {}] ipc.CallRunner(138): callId: 55 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:32822 deadline: 1732486591763, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94f3ccf9f8a638fa974e1f69627832ef, server=bd6592c37bb2,40087,1732486423713 2024-11-24T22:15:31,776 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732486521610.94f3ccf9f8a638fa974e1f69627832ef., hostname=bd6592c37bb2,40087,1732486423713, seqNum=5 , the old value is region=TestAcidGuarantees,,1732486521610.94f3ccf9f8a638fa974e1f69627832ef., hostname=bd6592c37bb2,40087,1732486423713, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94f3ccf9f8a638fa974e1f69627832ef, server=bd6592c37bb2,40087,1732486423713 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T22:15:31,776 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732486521610.94f3ccf9f8a638fa974e1f69627832ef., hostname=bd6592c37bb2,40087,1732486423713, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94f3ccf9f8a638fa974e1f69627832ef, server=bd6592c37bb2,40087,1732486423713 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T22:15:31,776 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732486521610.94f3ccf9f8a638fa974e1f69627832ef., hostname=bd6592c37bb2,40087,1732486423713, seqNum=5 because the exception is null or not the one we care about 2024-11-24T22:15:31,800 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45929 is added to blk_1073742210_1386 (size=12001) 2024-11-24T22:15:31,802 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=24.60 KB at sequenceid=120 (bloomFilter=true), to=hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/94f3ccf9f8a638fa974e1f69627832ef/.tmp/B/1c3c63e6b7a943afbb4d603545e4dd0a 2024-11-24T22:15:31,810 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/94f3ccf9f8a638fa974e1f69627832ef/.tmp/C/fc561c7bb18d48ffa58362322051fad8 is 50, key is test_row_0/C:col10/1732486530255/Put/seqid=0 2024-11-24T22:15:31,860 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45929 is added to blk_1073742211_1387 (size=12001) 2024-11-24T22:15:32,096 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40087 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94f3ccf9f8a638fa974e1f69627832ef, server=bd6592c37bb2,40087,1732486423713 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-24T22:15:32,096 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40087 {}] ipc.CallRunner(138): callId: 56 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:32822 deadline: 1732486592081, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94f3ccf9f8a638fa974e1f69627832ef, server=bd6592c37bb2,40087,1732486423713 2024-11-24T22:15:32,099 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732486521610.94f3ccf9f8a638fa974e1f69627832ef., hostname=bd6592c37bb2,40087,1732486423713, seqNum=5 , the old value is region=TestAcidGuarantees,,1732486521610.94f3ccf9f8a638fa974e1f69627832ef., hostname=bd6592c37bb2,40087,1732486423713, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94f3ccf9f8a638fa974e1f69627832ef, server=bd6592c37bb2,40087,1732486423713 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T22:15:32,099 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732486521610.94f3ccf9f8a638fa974e1f69627832ef., hostname=bd6592c37bb2,40087,1732486423713, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94f3ccf9f8a638fa974e1f69627832ef, server=bd6592c37bb2,40087,1732486423713 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T22:15:32,099 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732486521610.94f3ccf9f8a638fa974e1f69627832ef., hostname=bd6592c37bb2,40087,1732486423713, seqNum=5 because the exception is null or not the one we care about 2024-11-24T22:15:32,261 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=24.60 KB at sequenceid=120 (bloomFilter=true), to=hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/94f3ccf9f8a638fa974e1f69627832ef/.tmp/C/fc561c7bb18d48ffa58362322051fad8 2024-11-24T22:15:32,278 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/94f3ccf9f8a638fa974e1f69627832ef/.tmp/A/3e23a020d2d641a09079ca5e307d947e as hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/94f3ccf9f8a638fa974e1f69627832ef/A/3e23a020d2d641a09079ca5e307d947e 2024-11-24T22:15:32,290 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/94f3ccf9f8a638fa974e1f69627832ef/A/3e23a020d2d641a09079ca5e307d947e, entries=200, sequenceid=120, filesize=38.6 K 2024-11-24T22:15:32,291 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/94f3ccf9f8a638fa974e1f69627832ef/.tmp/B/1c3c63e6b7a943afbb4d603545e4dd0a as hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/94f3ccf9f8a638fa974e1f69627832ef/B/1c3c63e6b7a943afbb4d603545e4dd0a 2024-11-24T22:15:32,296 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/94f3ccf9f8a638fa974e1f69627832ef/B/1c3c63e6b7a943afbb4d603545e4dd0a, entries=150, sequenceid=120, filesize=11.7 K 2024-11-24T22:15:32,299 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/94f3ccf9f8a638fa974e1f69627832ef/.tmp/C/fc561c7bb18d48ffa58362322051fad8 as hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/94f3ccf9f8a638fa974e1f69627832ef/C/fc561c7bb18d48ffa58362322051fad8 2024-11-24T22:15:32,303 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/94f3ccf9f8a638fa974e1f69627832ef/C/fc561c7bb18d48ffa58362322051fad8, entries=150, sequenceid=120, filesize=11.7 K 2024-11-24T22:15:32,305 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~73.80 KB/75570, heapSize ~219.84 KB/225120, currentSize=100.63 KB/103050 for 94f3ccf9f8a638fa974e1f69627832ef in 1016ms, sequenceid=120, compaction requested=true 2024-11-24T22:15:32,305 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 94f3ccf9f8a638fa974e1f69627832ef: 2024-11-24T22:15:32,305 DEBUG [RS:0;bd6592c37bb2:40087-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-24T22:15:32,305 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 94f3ccf9f8a638fa974e1f69627832ef:A, priority=-2147483648, current under compaction store size is 1 2024-11-24T22:15:32,305 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-24T22:15:32,305 DEBUG [RS:0;bd6592c37bb2:40087-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-24T22:15:32,306 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 94f3ccf9f8a638fa974e1f69627832ef:B, priority=-2147483648, current under compaction store size is 2 2024-11-24T22:15:32,306 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-24T22:15:32,306 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 94f3ccf9f8a638fa974e1f69627832ef:C, priority=-2147483648, current under compaction store size is 3 2024-11-24T22:15:32,306 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-24T22:15:32,308 DEBUG [RS:0;bd6592c37bb2:40087-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 101665 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-24T22:15:32,308 DEBUG [RS:0;bd6592c37bb2:40087-longCompactions-0 {}] regionserver.HStore(1541): 94f3ccf9f8a638fa974e1f69627832ef/A is initiating minor compaction (all files) 2024-11-24T22:15:32,308 INFO [RS:0;bd6592c37bb2:40087-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 94f3ccf9f8a638fa974e1f69627832ef/A in TestAcidGuarantees,,1732486521610.94f3ccf9f8a638fa974e1f69627832ef. 2024-11-24T22:15:32,308 INFO [RS:0;bd6592c37bb2:40087-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/94f3ccf9f8a638fa974e1f69627832ef/A/83f546c8ac3a4c0dbfe435b59d1e9387, hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/94f3ccf9f8a638fa974e1f69627832ef/A/e78a5bea000d4b5088fbdfd1f6b3d2f8, hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/94f3ccf9f8a638fa974e1f69627832ef/A/3e23a020d2d641a09079ca5e307d947e] into tmpdir=hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/94f3ccf9f8a638fa974e1f69627832ef/.tmp, totalSize=99.3 K 2024-11-24T22:15:32,308 INFO [RS:0;bd6592c37bb2:40087-longCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=13 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1732486521610.94f3ccf9f8a638fa974e1f69627832ef. 2024-11-24T22:15:32,308 DEBUG [RS:0;bd6592c37bb2:40087-longCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1732486521610.94f3ccf9f8a638fa974e1f69627832ef. files: [hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/94f3ccf9f8a638fa974e1f69627832ef/A/83f546c8ac3a4c0dbfe435b59d1e9387, hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/94f3ccf9f8a638fa974e1f69627832ef/A/e78a5bea000d4b5088fbdfd1f6b3d2f8, hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/94f3ccf9f8a638fa974e1f69627832ef/A/3e23a020d2d641a09079ca5e307d947e] 2024-11-24T22:15:32,309 DEBUG [RS:0;bd6592c37bb2:40087-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36209 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-24T22:15:32,309 DEBUG [RS:0;bd6592c37bb2:40087-shortCompactions-0 {}] regionserver.HStore(1541): 94f3ccf9f8a638fa974e1f69627832ef/B is initiating minor compaction (all files) 2024-11-24T22:15:32,309 INFO [RS:0;bd6592c37bb2:40087-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 94f3ccf9f8a638fa974e1f69627832ef/B in TestAcidGuarantees,,1732486521610.94f3ccf9f8a638fa974e1f69627832ef. 2024-11-24T22:15:32,309 INFO [RS:0;bd6592c37bb2:40087-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/94f3ccf9f8a638fa974e1f69627832ef/B/d8beb157656c45efade39f49eb755238, hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/94f3ccf9f8a638fa974e1f69627832ef/B/97bd3de9d98b49909095a4474b63e142, hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/94f3ccf9f8a638fa974e1f69627832ef/B/1c3c63e6b7a943afbb4d603545e4dd0a] into tmpdir=hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/94f3ccf9f8a638fa974e1f69627832ef/.tmp, totalSize=35.4 K 2024-11-24T22:15:32,311 DEBUG [RS:0;bd6592c37bb2:40087-shortCompactions-0 {}] compactions.Compactor(225): Compacting d8beb157656c45efade39f49eb755238, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=82, earliestPutTs=1732486528887 2024-11-24T22:15:32,312 DEBUG [RS:0;bd6592c37bb2:40087-longCompactions-0 {}] compactions.Compactor(225): Compacting 83f546c8ac3a4c0dbfe435b59d1e9387, keycount=150, bloomtype=ROW, size=30.4 K, encoding=NONE, compression=NONE, seqNum=82, earliestPutTs=1732486528887 2024-11-24T22:15:32,312 DEBUG [RS:0;bd6592c37bb2:40087-shortCompactions-0 {}] compactions.Compactor(225): Compacting 97bd3de9d98b49909095a4474b63e142, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=104, earliestPutTs=1732486529052 2024-11-24T22:15:32,313 DEBUG [RS:0;bd6592c37bb2:40087-longCompactions-0 {}] compactions.Compactor(225): Compacting e78a5bea000d4b5088fbdfd1f6b3d2f8, keycount=150, bloomtype=ROW, size=30.2 K, encoding=NONE, compression=NONE, seqNum=104, earliestPutTs=1732486529052 2024-11-24T22:15:32,315 DEBUG [RS:0;bd6592c37bb2:40087-shortCompactions-0 {}] compactions.Compactor(225): Compacting 1c3c63e6b7a943afbb4d603545e4dd0a, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=120, earliestPutTs=1732486530255 2024-11-24T22:15:32,315 DEBUG [RS:0;bd6592c37bb2:40087-longCompactions-0 {}] compactions.Compactor(225): Compacting 3e23a020d2d641a09079ca5e307d947e, keycount=200, bloomtype=ROW, size=38.6 K, encoding=NONE, compression=NONE, seqNum=120, earliestPutTs=1732486530255 2024-11-24T22:15:32,323 INFO [RS:0;bd6592c37bb2:40087-longCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=94f3ccf9f8a638fa974e1f69627832ef] 2024-11-24T22:15:32,325 INFO [RS:0;bd6592c37bb2:40087-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 94f3ccf9f8a638fa974e1f69627832ef#B#compaction#326 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-24T22:15:32,325 DEBUG [RS:0;bd6592c37bb2:40087-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/94f3ccf9f8a638fa974e1f69627832ef/.tmp/B/77b19474d2d548f7916c60b8c12adad0 is 50, key is test_row_0/B:col10/1732486530255/Put/seqid=0 2024-11-24T22:15:32,325 DEBUG [RS:0;bd6592c37bb2:40087-longCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e20241124e84cafc810ef4ef0b3a7029017ea59a3_94f3ccf9f8a638fa974e1f69627832ef store=[table=TestAcidGuarantees family=A region=94f3ccf9f8a638fa974e1f69627832ef] 2024-11-24T22:15:32,328 DEBUG [RS:0;bd6592c37bb2:40087-longCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e20241124e84cafc810ef4ef0b3a7029017ea59a3_94f3ccf9f8a638fa974e1f69627832ef, store=[table=TestAcidGuarantees family=A region=94f3ccf9f8a638fa974e1f69627832ef] 2024-11-24T22:15:32,328 DEBUG [RS:0;bd6592c37bb2:40087-longCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241124e84cafc810ef4ef0b3a7029017ea59a3_94f3ccf9f8a638fa974e1f69627832ef because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=94f3ccf9f8a638fa974e1f69627832ef] 2024-11-24T22:15:32,342 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45929 is added to blk_1073742212_1388 (size=12309) 2024-11-24T22:15:32,343 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45929 is added to blk_1073742213_1389 (size=4469) 2024-11-24T22:15:32,352 DEBUG [RS:0;bd6592c37bb2:40087-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/94f3ccf9f8a638fa974e1f69627832ef/.tmp/B/77b19474d2d548f7916c60b8c12adad0 as hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/94f3ccf9f8a638fa974e1f69627832ef/B/77b19474d2d548f7916c60b8c12adad0 2024-11-24T22:15:32,357 INFO [RS:0;bd6592c37bb2:40087-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 94f3ccf9f8a638fa974e1f69627832ef/B of 94f3ccf9f8a638fa974e1f69627832ef into 77b19474d2d548f7916c60b8c12adad0(size=12.0 K), total size for store is 12.0 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-24T22:15:32,357 DEBUG [RS:0;bd6592c37bb2:40087-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 94f3ccf9f8a638fa974e1f69627832ef: 2024-11-24T22:15:32,357 INFO [RS:0;bd6592c37bb2:40087-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732486521610.94f3ccf9f8a638fa974e1f69627832ef., storeName=94f3ccf9f8a638fa974e1f69627832ef/B, priority=13, startTime=1732486532305; duration=0sec 2024-11-24T22:15:32,357 DEBUG [RS:0;bd6592c37bb2:40087-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-24T22:15:32,357 DEBUG [RS:0;bd6592c37bb2:40087-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 94f3ccf9f8a638fa974e1f69627832ef:B 2024-11-24T22:15:32,357 DEBUG [RS:0;bd6592c37bb2:40087-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-24T22:15:32,358 DEBUG [RS:0;bd6592c37bb2:40087-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 48142 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-24T22:15:32,358 DEBUG [RS:0;bd6592c37bb2:40087-shortCompactions-0 {}] regionserver.HStore(1541): 94f3ccf9f8a638fa974e1f69627832ef/C is initiating minor compaction (all files) 2024-11-24T22:15:32,358 INFO [RS:0;bd6592c37bb2:40087-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 94f3ccf9f8a638fa974e1f69627832ef/C in TestAcidGuarantees,,1732486521610.94f3ccf9f8a638fa974e1f69627832ef. 2024-11-24T22:15:32,358 INFO [RS:0;bd6592c37bb2:40087-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/94f3ccf9f8a638fa974e1f69627832ef/C/267a2d67eb91491c826a9d1801fe9d29, hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/94f3ccf9f8a638fa974e1f69627832ef/C/9b339451e3df4507bf90584aa37d34bd, hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/94f3ccf9f8a638fa974e1f69627832ef/C/f90e75ec41424220862d0a6950395ea9, hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/94f3ccf9f8a638fa974e1f69627832ef/C/fc561c7bb18d48ffa58362322051fad8] into tmpdir=hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/94f3ccf9f8a638fa974e1f69627832ef/.tmp, totalSize=47.0 K 2024-11-24T22:15:32,359 DEBUG [RS:0;bd6592c37bb2:40087-shortCompactions-0 {}] compactions.Compactor(225): Compacting 267a2d67eb91491c826a9d1801fe9d29, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=69, earliestPutTs=1732486528203 2024-11-24T22:15:32,359 DEBUG [RS:0;bd6592c37bb2:40087-shortCompactions-0 {}] compactions.Compactor(225): Compacting 9b339451e3df4507bf90584aa37d34bd, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=82, earliestPutTs=1732486528887 2024-11-24T22:15:32,359 DEBUG [RS:0;bd6592c37bb2:40087-shortCompactions-0 {}] compactions.Compactor(225): Compacting f90e75ec41424220862d0a6950395ea9, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=104, earliestPutTs=1732486529052 2024-11-24T22:15:32,360 DEBUG [RS:0;bd6592c37bb2:40087-shortCompactions-0 {}] compactions.Compactor(225): Compacting fc561c7bb18d48ffa58362322051fad8, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=120, earliestPutTs=1732486530255 2024-11-24T22:15:32,375 INFO [RS:0;bd6592c37bb2:40087-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 94f3ccf9f8a638fa974e1f69627832ef#C#compaction#328 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-24T22:15:32,375 DEBUG [RS:0;bd6592c37bb2:40087-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/94f3ccf9f8a638fa974e1f69627832ef/.tmp/C/cb3ac673639340e892a044d258cbd444 is 50, key is test_row_0/C:col10/1732486530255/Put/seqid=0 2024-11-24T22:15:32,398 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45929 is added to blk_1073742214_1390 (size=12275) 2024-11-24T22:15:32,410 DEBUG [RS:0;bd6592c37bb2:40087-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/94f3ccf9f8a638fa974e1f69627832ef/.tmp/C/cb3ac673639340e892a044d258cbd444 as hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/94f3ccf9f8a638fa974e1f69627832ef/C/cb3ac673639340e892a044d258cbd444 2024-11-24T22:15:32,414 INFO [RS:0;bd6592c37bb2:40087-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 94f3ccf9f8a638fa974e1f69627832ef/C of 94f3ccf9f8a638fa974e1f69627832ef into cb3ac673639340e892a044d258cbd444(size=12.0 K), total size for store is 12.0 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-24T22:15:32,414 DEBUG [RS:0;bd6592c37bb2:40087-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 94f3ccf9f8a638fa974e1f69627832ef: 2024-11-24T22:15:32,414 INFO [RS:0;bd6592c37bb2:40087-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732486521610.94f3ccf9f8a638fa974e1f69627832ef., storeName=94f3ccf9f8a638fa974e1f69627832ef/C, priority=12, startTime=1732486532306; duration=0sec 2024-11-24T22:15:32,414 DEBUG [RS:0;bd6592c37bb2:40087-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-24T22:15:32,415 DEBUG [RS:0;bd6592c37bb2:40087-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 94f3ccf9f8a638fa974e1f69627832ef:C 2024-11-24T22:15:32,442 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40087 {}] regionserver.HRegion(8855): Flush requested on 94f3ccf9f8a638fa974e1f69627832ef 2024-11-24T22:15:32,443 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 94f3ccf9f8a638fa974e1f69627832ef 3/3 column families, dataSize=107.34 KB heapSize=319.50 KB 2024-11-24T22:15:32,443 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 94f3ccf9f8a638fa974e1f69627832ef, store=A 2024-11-24T22:15:32,443 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-24T22:15:32,443 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 94f3ccf9f8a638fa974e1f69627832ef, store=B 2024-11-24T22:15:32,443 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-24T22:15:32,443 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 94f3ccf9f8a638fa974e1f69627832ef, store=C 2024-11-24T22:15:32,443 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-24T22:15:32,453 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024112488d65ced9a884882a2ed516a7cb18106_94f3ccf9f8a638fa974e1f69627832ef is 50, key is test_row_0/A:col10/1732486531406/Put/seqid=0 2024-11-24T22:15:32,493 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45929 is added to blk_1073742215_1391 (size=14794) 2024-11-24T22:15:32,504 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40087 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94f3ccf9f8a638fa974e1f69627832ef, server=bd6592c37bb2,40087,1732486423713 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-24T22:15:32,504 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40087 {}] ipc.CallRunner(138): callId: 30 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:32832 deadline: 1732486592472, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94f3ccf9f8a638fa974e1f69627832ef, server=bd6592c37bb2,40087,1732486423713 2024-11-24T22:15:32,505 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40087 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94f3ccf9f8a638fa974e1f69627832ef, server=bd6592c37bb2,40087,1732486423713 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-24T22:15:32,505 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40087 {}] ipc.CallRunner(138): callId: 44 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:32846 deadline: 1732486592485, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94f3ccf9f8a638fa974e1f69627832ef, server=bd6592c37bb2,40087,1732486423713 2024-11-24T22:15:32,505 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732486521610.94f3ccf9f8a638fa974e1f69627832ef., hostname=bd6592c37bb2,40087,1732486423713, seqNum=5 , the old value is region=TestAcidGuarantees,,1732486521610.94f3ccf9f8a638fa974e1f69627832ef., hostname=bd6592c37bb2,40087,1732486423713, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94f3ccf9f8a638fa974e1f69627832ef, server=bd6592c37bb2,40087,1732486423713 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T22:15:32,505 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732486521610.94f3ccf9f8a638fa974e1f69627832ef., hostname=bd6592c37bb2,40087,1732486423713, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94f3ccf9f8a638fa974e1f69627832ef, server=bd6592c37bb2,40087,1732486423713 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T22:15:32,505 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732486521610.94f3ccf9f8a638fa974e1f69627832ef., hostname=bd6592c37bb2,40087,1732486423713, seqNum=5 because the exception is null or not the one we care about 2024-11-24T22:15:32,506 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732486521610.94f3ccf9f8a638fa974e1f69627832ef., hostname=bd6592c37bb2,40087,1732486423713, seqNum=5 , the old value is region=TestAcidGuarantees,,1732486521610.94f3ccf9f8a638fa974e1f69627832ef., hostname=bd6592c37bb2,40087,1732486423713, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94f3ccf9f8a638fa974e1f69627832ef, server=bd6592c37bb2,40087,1732486423713 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T22:15:32,506 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732486521610.94f3ccf9f8a638fa974e1f69627832ef., hostname=bd6592c37bb2,40087,1732486423713, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94f3ccf9f8a638fa974e1f69627832ef, server=bd6592c37bb2,40087,1732486423713 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T22:15:32,506 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732486521610.94f3ccf9f8a638fa974e1f69627832ef., hostname=bd6592c37bb2,40087,1732486423713, seqNum=5 because the exception is null or not the one we care about 2024-11-24T22:15:32,515 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T22:15:32,518 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40087 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94f3ccf9f8a638fa974e1f69627832ef, server=bd6592c37bb2,40087,1732486423713 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-24T22:15:32,518 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40087 {}] ipc.CallRunner(138): callId: 47 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:32810 deadline: 1732486592504, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94f3ccf9f8a638fa974e1f69627832ef, server=bd6592c37bb2,40087,1732486423713 2024-11-24T22:15:32,518 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40087 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94f3ccf9f8a638fa974e1f69627832ef, server=bd6592c37bb2,40087,1732486423713 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-24T22:15:32,518 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40087 {}] ipc.CallRunner(138): callId: 47 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:32774 deadline: 1732486592505, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94f3ccf9f8a638fa974e1f69627832ef, server=bd6592c37bb2,40087,1732486423713 2024-11-24T22:15:32,519 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732486521610.94f3ccf9f8a638fa974e1f69627832ef., hostname=bd6592c37bb2,40087,1732486423713, seqNum=5 , the old value is region=TestAcidGuarantees,,1732486521610.94f3ccf9f8a638fa974e1f69627832ef., hostname=bd6592c37bb2,40087,1732486423713, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94f3ccf9f8a638fa974e1f69627832ef, server=bd6592c37bb2,40087,1732486423713 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T22:15:32,519 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732486521610.94f3ccf9f8a638fa974e1f69627832ef., hostname=bd6592c37bb2,40087,1732486423713, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94f3ccf9f8a638fa974e1f69627832ef, server=bd6592c37bb2,40087,1732486423713 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T22:15:32,519 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732486521610.94f3ccf9f8a638fa974e1f69627832ef., hostname=bd6592c37bb2,40087,1732486423713, seqNum=5 because the exception is null or not the one we care about 2024-11-24T22:15:32,519 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732486521610.94f3ccf9f8a638fa974e1f69627832ef., hostname=bd6592c37bb2,40087,1732486423713, seqNum=5 , the old value is region=TestAcidGuarantees,,1732486521610.94f3ccf9f8a638fa974e1f69627832ef., hostname=bd6592c37bb2,40087,1732486423713, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94f3ccf9f8a638fa974e1f69627832ef, server=bd6592c37bb2,40087,1732486423713 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T22:15:32,519 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732486521610.94f3ccf9f8a638fa974e1f69627832ef., hostname=bd6592c37bb2,40087,1732486423713, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94f3ccf9f8a638fa974e1f69627832ef, server=bd6592c37bb2,40087,1732486423713 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T22:15:32,519 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732486521610.94f3ccf9f8a638fa974e1f69627832ef., hostname=bd6592c37bb2,40087,1732486423713, seqNum=5 because the exception is null or not the one we care about 2024-11-24T22:15:32,520 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024112488d65ced9a884882a2ed516a7cb18106_94f3ccf9f8a638fa974e1f69627832ef to hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024112488d65ced9a884882a2ed516a7cb18106_94f3ccf9f8a638fa974e1f69627832ef 2024-11-24T22:15:32,523 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/94f3ccf9f8a638fa974e1f69627832ef/.tmp/A/697a1699f64d45fcb65780147612779b, store: [table=TestAcidGuarantees family=A region=94f3ccf9f8a638fa974e1f69627832ef] 2024-11-24T22:15:32,523 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/94f3ccf9f8a638fa974e1f69627832ef/.tmp/A/697a1699f64d45fcb65780147612779b is 175, key is test_row_0/A:col10/1732486531406/Put/seqid=0 2024-11-24T22:15:32,544 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45929 is added to blk_1073742216_1392 (size=39749) 2024-11-24T22:15:32,620 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40087 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94f3ccf9f8a638fa974e1f69627832ef, server=bd6592c37bb2,40087,1732486423713 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-24T22:15:32,620 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40087 {}] ipc.CallRunner(138): callId: 57 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:32822 deadline: 1732486592611, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94f3ccf9f8a638fa974e1f69627832ef, server=bd6592c37bb2,40087,1732486423713 2024-11-24T22:15:32,620 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40087 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94f3ccf9f8a638fa974e1f69627832ef, server=bd6592c37bb2,40087,1732486423713 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-24T22:15:32,620 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732486521610.94f3ccf9f8a638fa974e1f69627832ef., hostname=bd6592c37bb2,40087,1732486423713, seqNum=5 , the old value is region=TestAcidGuarantees,,1732486521610.94f3ccf9f8a638fa974e1f69627832ef., hostname=bd6592c37bb2,40087,1732486423713, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94f3ccf9f8a638fa974e1f69627832ef, server=bd6592c37bb2,40087,1732486423713 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T22:15:32,620 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732486521610.94f3ccf9f8a638fa974e1f69627832ef., hostname=bd6592c37bb2,40087,1732486423713, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94f3ccf9f8a638fa974e1f69627832ef, server=bd6592c37bb2,40087,1732486423713 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T22:15:32,620 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40087 {}] ipc.CallRunner(138): callId: 45 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:32846 deadline: 1732486592611, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94f3ccf9f8a638fa974e1f69627832ef, server=bd6592c37bb2,40087,1732486423713 2024-11-24T22:15:32,620 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732486521610.94f3ccf9f8a638fa974e1f69627832ef., hostname=bd6592c37bb2,40087,1732486423713, seqNum=5 because the exception is null or not the one we care about 2024-11-24T22:15:32,621 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40087 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94f3ccf9f8a638fa974e1f69627832ef, server=bd6592c37bb2,40087,1732486423713 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-24T22:15:32,621 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40087 {}] ipc.CallRunner(138): callId: 31 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:32832 deadline: 1732486592611, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94f3ccf9f8a638fa974e1f69627832ef, server=bd6592c37bb2,40087,1732486423713 2024-11-24T22:15:32,621 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732486521610.94f3ccf9f8a638fa974e1f69627832ef., hostname=bd6592c37bb2,40087,1732486423713, seqNum=5 , the old value is region=TestAcidGuarantees,,1732486521610.94f3ccf9f8a638fa974e1f69627832ef., hostname=bd6592c37bb2,40087,1732486423713, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94f3ccf9f8a638fa974e1f69627832ef, server=bd6592c37bb2,40087,1732486423713 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T22:15:32,621 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732486521610.94f3ccf9f8a638fa974e1f69627832ef., hostname=bd6592c37bb2,40087,1732486423713, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94f3ccf9f8a638fa974e1f69627832ef, server=bd6592c37bb2,40087,1732486423713 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T22:15:32,621 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732486521610.94f3ccf9f8a638fa974e1f69627832ef., hostname=bd6592c37bb2,40087,1732486423713, seqNum=5 because the exception is null or not the one we care about 2024-11-24T22:15:32,621 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732486521610.94f3ccf9f8a638fa974e1f69627832ef., hostname=bd6592c37bb2,40087,1732486423713, seqNum=5 , the old value is region=TestAcidGuarantees,,1732486521610.94f3ccf9f8a638fa974e1f69627832ef., hostname=bd6592c37bb2,40087,1732486423713, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94f3ccf9f8a638fa974e1f69627832ef, server=bd6592c37bb2,40087,1732486423713 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T22:15:32,621 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732486521610.94f3ccf9f8a638fa974e1f69627832ef., hostname=bd6592c37bb2,40087,1732486423713, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94f3ccf9f8a638fa974e1f69627832ef, server=bd6592c37bb2,40087,1732486423713 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T22:15:32,621 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732486521610.94f3ccf9f8a638fa974e1f69627832ef., hostname=bd6592c37bb2,40087,1732486423713, seqNum=5 because the exception is null or not the one we care about 2024-11-24T22:15:32,629 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40087 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94f3ccf9f8a638fa974e1f69627832ef, server=bd6592c37bb2,40087,1732486423713 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-24T22:15:32,630 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40087 {}] ipc.CallRunner(138): callId: 48 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:32774 deadline: 1732486592620, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94f3ccf9f8a638fa974e1f69627832ef, server=bd6592c37bb2,40087,1732486423713 2024-11-24T22:15:32,630 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732486521610.94f3ccf9f8a638fa974e1f69627832ef., hostname=bd6592c37bb2,40087,1732486423713, seqNum=5 , the old value is region=TestAcidGuarantees,,1732486521610.94f3ccf9f8a638fa974e1f69627832ef., hostname=bd6592c37bb2,40087,1732486423713, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94f3ccf9f8a638fa974e1f69627832ef, server=bd6592c37bb2,40087,1732486423713 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T22:15:32,630 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40087 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94f3ccf9f8a638fa974e1f69627832ef, server=bd6592c37bb2,40087,1732486423713 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-24T22:15:32,630 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732486521610.94f3ccf9f8a638fa974e1f69627832ef., hostname=bd6592c37bb2,40087,1732486423713, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94f3ccf9f8a638fa974e1f69627832ef, server=bd6592c37bb2,40087,1732486423713 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T22:15:32,630 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732486521610.94f3ccf9f8a638fa974e1f69627832ef., hostname=bd6592c37bb2,40087,1732486423713, seqNum=5 because the exception is null or not the one we care about 2024-11-24T22:15:32,630 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40087 {}] ipc.CallRunner(138): callId: 48 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:32810 deadline: 1732486592620, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94f3ccf9f8a638fa974e1f69627832ef, server=bd6592c37bb2,40087,1732486423713 2024-11-24T22:15:32,630 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732486521610.94f3ccf9f8a638fa974e1f69627832ef., hostname=bd6592c37bb2,40087,1732486423713, seqNum=5 , the old value is region=TestAcidGuarantees,,1732486521610.94f3ccf9f8a638fa974e1f69627832ef., hostname=bd6592c37bb2,40087,1732486423713, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94f3ccf9f8a638fa974e1f69627832ef, server=bd6592c37bb2,40087,1732486423713 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T22:15:32,631 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732486521610.94f3ccf9f8a638fa974e1f69627832ef., hostname=bd6592c37bb2,40087,1732486423713, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94f3ccf9f8a638fa974e1f69627832ef, server=bd6592c37bb2,40087,1732486423713 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T22:15:32,631 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732486521610.94f3ccf9f8a638fa974e1f69627832ef., hostname=bd6592c37bb2,40087,1732486423713, seqNum=5 because the exception is null or not the one we care about 2024-11-24T22:15:32,747 INFO [RS:0;bd6592c37bb2:40087-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 94f3ccf9f8a638fa974e1f69627832ef#A#compaction#327 average throughput is 0.06 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-24T22:15:32,747 DEBUG [RS:0;bd6592c37bb2:40087-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/94f3ccf9f8a638fa974e1f69627832ef/.tmp/A/fce104982b484448966f2a223c8fc392 is 175, key is test_row_0/A:col10/1732486530255/Put/seqid=0 2024-11-24T22:15:32,752 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45929 is added to blk_1073742217_1393 (size=31263) 2024-11-24T22:15:32,838 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40087 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94f3ccf9f8a638fa974e1f69627832ef, server=bd6592c37bb2,40087,1732486423713 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-24T22:15:32,839 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40087 {}] ipc.CallRunner(138): callId: 46 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:32846 deadline: 1732486592831, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94f3ccf9f8a638fa974e1f69627832ef, server=bd6592c37bb2,40087,1732486423713 2024-11-24T22:15:32,839 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40087 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94f3ccf9f8a638fa974e1f69627832ef, server=bd6592c37bb2,40087,1732486423713 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-24T22:15:32,839 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40087 {}] ipc.CallRunner(138): callId: 32 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:32832 deadline: 1732486592831, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94f3ccf9f8a638fa974e1f69627832ef, server=bd6592c37bb2,40087,1732486423713 2024-11-24T22:15:32,839 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732486521610.94f3ccf9f8a638fa974e1f69627832ef., hostname=bd6592c37bb2,40087,1732486423713, seqNum=5 , the old value is region=TestAcidGuarantees,,1732486521610.94f3ccf9f8a638fa974e1f69627832ef., hostname=bd6592c37bb2,40087,1732486423713, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94f3ccf9f8a638fa974e1f69627832ef, server=bd6592c37bb2,40087,1732486423713 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T22:15:32,839 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732486521610.94f3ccf9f8a638fa974e1f69627832ef., hostname=bd6592c37bb2,40087,1732486423713, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94f3ccf9f8a638fa974e1f69627832ef, server=bd6592c37bb2,40087,1732486423713 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T22:15:32,839 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732486521610.94f3ccf9f8a638fa974e1f69627832ef., hostname=bd6592c37bb2,40087,1732486423713, seqNum=5 because the exception is null or not the one we care about 2024-11-24T22:15:32,839 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732486521610.94f3ccf9f8a638fa974e1f69627832ef., hostname=bd6592c37bb2,40087,1732486423713, seqNum=5 , the old value is region=TestAcidGuarantees,,1732486521610.94f3ccf9f8a638fa974e1f69627832ef., hostname=bd6592c37bb2,40087,1732486423713, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94f3ccf9f8a638fa974e1f69627832ef, server=bd6592c37bb2,40087,1732486423713 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T22:15:32,839 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732486521610.94f3ccf9f8a638fa974e1f69627832ef., hostname=bd6592c37bb2,40087,1732486423713, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94f3ccf9f8a638fa974e1f69627832ef, server=bd6592c37bb2,40087,1732486423713 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T22:15:32,839 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732486521610.94f3ccf9f8a638fa974e1f69627832ef., hostname=bd6592c37bb2,40087,1732486423713, seqNum=5 because the exception is null or not the one we care about 2024-11-24T22:15:32,851 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40087 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94f3ccf9f8a638fa974e1f69627832ef, server=bd6592c37bb2,40087,1732486423713 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-24T22:15:32,851 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40087 {}] ipc.CallRunner(138): callId: 49 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:32774 deadline: 1732486592840, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94f3ccf9f8a638fa974e1f69627832ef, server=bd6592c37bb2,40087,1732486423713 2024-11-24T22:15:32,851 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40087 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94f3ccf9f8a638fa974e1f69627832ef, server=bd6592c37bb2,40087,1732486423713 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-24T22:15:32,851 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40087 {}] ipc.CallRunner(138): callId: 49 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:32810 deadline: 1732486592841, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94f3ccf9f8a638fa974e1f69627832ef, server=bd6592c37bb2,40087,1732486423713 2024-11-24T22:15:32,851 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732486521610.94f3ccf9f8a638fa974e1f69627832ef., hostname=bd6592c37bb2,40087,1732486423713, seqNum=5 , the old value is region=TestAcidGuarantees,,1732486521610.94f3ccf9f8a638fa974e1f69627832ef., hostname=bd6592c37bb2,40087,1732486423713, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94f3ccf9f8a638fa974e1f69627832ef, server=bd6592c37bb2,40087,1732486423713 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T22:15:32,851 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732486521610.94f3ccf9f8a638fa974e1f69627832ef., hostname=bd6592c37bb2,40087,1732486423713, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94f3ccf9f8a638fa974e1f69627832ef, server=bd6592c37bb2,40087,1732486423713 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T22:15:32,851 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732486521610.94f3ccf9f8a638fa974e1f69627832ef., hostname=bd6592c37bb2,40087,1732486423713, seqNum=5 because the exception is null or not the one we care about 2024-11-24T22:15:32,851 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732486521610.94f3ccf9f8a638fa974e1f69627832ef., hostname=bd6592c37bb2,40087,1732486423713, seqNum=5 , the old value is region=TestAcidGuarantees,,1732486521610.94f3ccf9f8a638fa974e1f69627832ef., hostname=bd6592c37bb2,40087,1732486423713, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94f3ccf9f8a638fa974e1f69627832ef, server=bd6592c37bb2,40087,1732486423713 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T22:15:32,851 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732486521610.94f3ccf9f8a638fa974e1f69627832ef., hostname=bd6592c37bb2,40087,1732486423713, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94f3ccf9f8a638fa974e1f69627832ef, server=bd6592c37bb2,40087,1732486423713 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T22:15:32,851 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732486521610.94f3ccf9f8a638fa974e1f69627832ef., hostname=bd6592c37bb2,40087,1732486423713, seqNum=5 because the exception is null or not the one we care about 2024-11-24T22:15:32,945 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=141, memsize=35.8 K, hasBloomFilter=true, into tmp file hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/94f3ccf9f8a638fa974e1f69627832ef/.tmp/A/697a1699f64d45fcb65780147612779b 2024-11-24T22:15:32,952 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/94f3ccf9f8a638fa974e1f69627832ef/.tmp/B/d141ed51dbd0448f9084a2a5552ef064 is 50, key is test_row_0/B:col10/1732486531406/Put/seqid=0 2024-11-24T22:15:32,956 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45929 is added to blk_1073742218_1394 (size=12151) 2024-11-24T22:15:33,037 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37701 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=99 2024-11-24T22:15:33,038 INFO [RPCClient-NioEventLoopGroup-4-8 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-24T22:15:33,038 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37701 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-24T22:15:33,039 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37701 {}] procedure2.ProcedureExecutor(1139): Stored pid=101, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=101, table=TestAcidGuarantees 2024-11-24T22:15:33,040 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37701 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=101 2024-11-24T22:15:33,040 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=101, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=101, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-24T22:15:33,041 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=101, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=101, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-24T22:15:33,041 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=102, ppid=101, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-24T22:15:33,151 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37701 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=101 2024-11-24T22:15:33,157 DEBUG [RS:0;bd6592c37bb2:40087-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/94f3ccf9f8a638fa974e1f69627832ef/.tmp/A/fce104982b484448966f2a223c8fc392 as hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/94f3ccf9f8a638fa974e1f69627832ef/A/fce104982b484448966f2a223c8fc392 2024-11-24T22:15:33,161 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40087 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94f3ccf9f8a638fa974e1f69627832ef, server=bd6592c37bb2,40087,1732486423713 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-24T22:15:33,161 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40087 {}] ipc.CallRunner(138): callId: 47 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:32846 deadline: 1732486593151, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94f3ccf9f8a638fa974e1f69627832ef, server=bd6592c37bb2,40087,1732486423713 2024-11-24T22:15:33,161 INFO [RS:0;bd6592c37bb2:40087-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 94f3ccf9f8a638fa974e1f69627832ef/A of 94f3ccf9f8a638fa974e1f69627832ef into fce104982b484448966f2a223c8fc392(size=30.5 K), total size for store is 30.5 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-24T22:15:33,161 DEBUG [RS:0;bd6592c37bb2:40087-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 94f3ccf9f8a638fa974e1f69627832ef: 2024-11-24T22:15:33,161 INFO [RS:0;bd6592c37bb2:40087-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732486521610.94f3ccf9f8a638fa974e1f69627832ef., storeName=94f3ccf9f8a638fa974e1f69627832ef/A, priority=13, startTime=1732486532305; duration=0sec 2024-11-24T22:15:33,161 DEBUG [RS:0;bd6592c37bb2:40087-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-24T22:15:33,161 DEBUG [RS:0;bd6592c37bb2:40087-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 94f3ccf9f8a638fa974e1f69627832ef:A 2024-11-24T22:15:33,161 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732486521610.94f3ccf9f8a638fa974e1f69627832ef., hostname=bd6592c37bb2,40087,1732486423713, seqNum=5 , the old value is region=TestAcidGuarantees,,1732486521610.94f3ccf9f8a638fa974e1f69627832ef., hostname=bd6592c37bb2,40087,1732486423713, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94f3ccf9f8a638fa974e1f69627832ef, server=bd6592c37bb2,40087,1732486423713 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T22:15:33,161 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40087 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94f3ccf9f8a638fa974e1f69627832ef, server=bd6592c37bb2,40087,1732486423713 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-24T22:15:33,161 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732486521610.94f3ccf9f8a638fa974e1f69627832ef., hostname=bd6592c37bb2,40087,1732486423713, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94f3ccf9f8a638fa974e1f69627832ef, server=bd6592c37bb2,40087,1732486423713 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T22:15:33,161 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40087 {}] ipc.CallRunner(138): callId: 33 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:32832 deadline: 1732486593151, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94f3ccf9f8a638fa974e1f69627832ef, server=bd6592c37bb2,40087,1732486423713 2024-11-24T22:15:33,161 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732486521610.94f3ccf9f8a638fa974e1f69627832ef., hostname=bd6592c37bb2,40087,1732486423713, seqNum=5 because the exception is null or not the one we care about 2024-11-24T22:15:33,162 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732486521610.94f3ccf9f8a638fa974e1f69627832ef., hostname=bd6592c37bb2,40087,1732486423713, seqNum=5 , the old value is region=TestAcidGuarantees,,1732486521610.94f3ccf9f8a638fa974e1f69627832ef., hostname=bd6592c37bb2,40087,1732486423713, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94f3ccf9f8a638fa974e1f69627832ef, server=bd6592c37bb2,40087,1732486423713 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T22:15:33,162 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732486521610.94f3ccf9f8a638fa974e1f69627832ef., hostname=bd6592c37bb2,40087,1732486423713, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94f3ccf9f8a638fa974e1f69627832ef, server=bd6592c37bb2,40087,1732486423713 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T22:15:33,162 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732486521610.94f3ccf9f8a638fa974e1f69627832ef., hostname=bd6592c37bb2,40087,1732486423713, seqNum=5 because the exception is null or not the one we care about 2024-11-24T22:15:33,167 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40087 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94f3ccf9f8a638fa974e1f69627832ef, server=bd6592c37bb2,40087,1732486423713 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-24T22:15:33,167 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40087 {}] ipc.CallRunner(138): callId: 50 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:32774 deadline: 1732486593160, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94f3ccf9f8a638fa974e1f69627832ef, server=bd6592c37bb2,40087,1732486423713 2024-11-24T22:15:33,167 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40087 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94f3ccf9f8a638fa974e1f69627832ef, server=bd6592c37bb2,40087,1732486423713 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-24T22:15:33,167 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40087 {}] ipc.CallRunner(138): callId: 50 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:32810 deadline: 1732486593161, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94f3ccf9f8a638fa974e1f69627832ef, server=bd6592c37bb2,40087,1732486423713 2024-11-24T22:15:33,167 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732486521610.94f3ccf9f8a638fa974e1f69627832ef., hostname=bd6592c37bb2,40087,1732486423713, seqNum=5 , the old value is region=TestAcidGuarantees,,1732486521610.94f3ccf9f8a638fa974e1f69627832ef., hostname=bd6592c37bb2,40087,1732486423713, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94f3ccf9f8a638fa974e1f69627832ef, server=bd6592c37bb2,40087,1732486423713 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T22:15:33,167 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732486521610.94f3ccf9f8a638fa974e1f69627832ef., hostname=bd6592c37bb2,40087,1732486423713, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94f3ccf9f8a638fa974e1f69627832ef, server=bd6592c37bb2,40087,1732486423713 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T22:15:33,167 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732486521610.94f3ccf9f8a638fa974e1f69627832ef., hostname=bd6592c37bb2,40087,1732486423713, seqNum=5 because the exception is null or not the one we care about 2024-11-24T22:15:33,167 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732486521610.94f3ccf9f8a638fa974e1f69627832ef., hostname=bd6592c37bb2,40087,1732486423713, seqNum=5 , the old value is region=TestAcidGuarantees,,1732486521610.94f3ccf9f8a638fa974e1f69627832ef., hostname=bd6592c37bb2,40087,1732486423713, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94f3ccf9f8a638fa974e1f69627832ef, server=bd6592c37bb2,40087,1732486423713 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T22:15:33,168 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732486521610.94f3ccf9f8a638fa974e1f69627832ef., hostname=bd6592c37bb2,40087,1732486423713, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94f3ccf9f8a638fa974e1f69627832ef, server=bd6592c37bb2,40087,1732486423713 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T22:15:33,168 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732486521610.94f3ccf9f8a638fa974e1f69627832ef., hostname=bd6592c37bb2,40087,1732486423713, seqNum=5 because the exception is null or not the one we care about 2024-11-24T22:15:33,193 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=40087 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=102 2024-11-24T22:15:33,193 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bd6592c37bb2:0-2 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732486521610.94f3ccf9f8a638fa974e1f69627832ef. 2024-11-24T22:15:33,193 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bd6592c37bb2:0-2 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732486521610.94f3ccf9f8a638fa974e1f69627832ef. as already flushing 2024-11-24T22:15:33,193 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bd6592c37bb2:0-2 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732486521610.94f3ccf9f8a638fa974e1f69627832ef. 2024-11-24T22:15:33,193 ERROR [RS_FLUSH_OPERATIONS-regionserver/bd6592c37bb2:0-2 {event_type=RS_FLUSH_REGIONS, pid=102}] handler.RSProcedureHandler(58): pid=102 java.io.IOException: Unable to complete flush {ENCODED => 94f3ccf9f8a638fa974e1f69627832ef, NAME => 'TestAcidGuarantees,,1732486521610.94f3ccf9f8a638fa974e1f69627832ef.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T22:15:33,193 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bd6592c37bb2:0-2 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=102 java.io.IOException: Unable to complete flush {ENCODED => 94f3ccf9f8a638fa974e1f69627832ef, NAME => 'TestAcidGuarantees,,1732486521610.94f3ccf9f8a638fa974e1f69627832ef.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T22:15:33,194 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37701 {}] master.HMaster(4177): Remote procedure failed, pid=102 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 94f3ccf9f8a638fa974e1f69627832ef, NAME => 'TestAcidGuarantees,,1732486521610.94f3ccf9f8a638fa974e1f69627832ef.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 94f3ccf9f8a638fa974e1f69627832ef, NAME => 'TestAcidGuarantees,,1732486521610.94f3ccf9f8a638fa974e1f69627832ef.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T22:15:33,346 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=40087 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=102 2024-11-24T22:15:33,346 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bd6592c37bb2:0-0 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732486521610.94f3ccf9f8a638fa974e1f69627832ef. 2024-11-24T22:15:33,346 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bd6592c37bb2:0-0 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732486521610.94f3ccf9f8a638fa974e1f69627832ef. as already flushing 2024-11-24T22:15:33,346 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bd6592c37bb2:0-0 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732486521610.94f3ccf9f8a638fa974e1f69627832ef. 2024-11-24T22:15:33,346 ERROR [RS_FLUSH_OPERATIONS-regionserver/bd6592c37bb2:0-0 {event_type=RS_FLUSH_REGIONS, pid=102}] handler.RSProcedureHandler(58): pid=102 java.io.IOException: Unable to complete flush {ENCODED => 94f3ccf9f8a638fa974e1f69627832ef, NAME => 'TestAcidGuarantees,,1732486521610.94f3ccf9f8a638fa974e1f69627832ef.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T22:15:33,346 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bd6592c37bb2:0-0 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=102 java.io.IOException: Unable to complete flush {ENCODED => 94f3ccf9f8a638fa974e1f69627832ef, NAME => 'TestAcidGuarantees,,1732486521610.94f3ccf9f8a638fa974e1f69627832ef.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T22:15:33,347 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37701 {}] master.HMaster(4177): Remote procedure failed, pid=102 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 94f3ccf9f8a638fa974e1f69627832ef, NAME => 'TestAcidGuarantees,,1732486521610.94f3ccf9f8a638fa974e1f69627832ef.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 94f3ccf9f8a638fa974e1f69627832ef, NAME => 'TestAcidGuarantees,,1732486521610.94f3ccf9f8a638fa974e1f69627832ef.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T22:15:33,356 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=35.78 KB at sequenceid=141 (bloomFilter=true), to=hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/94f3ccf9f8a638fa974e1f69627832ef/.tmp/B/d141ed51dbd0448f9084a2a5552ef064 2024-11-24T22:15:33,361 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37701 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=101 2024-11-24T22:15:33,365 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/94f3ccf9f8a638fa974e1f69627832ef/.tmp/C/a3395d0e4c1c43a98e2163a3d14415df is 50, key is test_row_0/C:col10/1732486531406/Put/seqid=0 2024-11-24T22:15:33,368 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45929 is added to blk_1073742219_1395 (size=12151) 2024-11-24T22:15:33,498 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=40087 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=102 2024-11-24T22:15:33,499 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bd6592c37bb2:0-1 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732486521610.94f3ccf9f8a638fa974e1f69627832ef. 2024-11-24T22:15:33,499 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bd6592c37bb2:0-1 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732486521610.94f3ccf9f8a638fa974e1f69627832ef. as already flushing 2024-11-24T22:15:33,499 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bd6592c37bb2:0-1 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732486521610.94f3ccf9f8a638fa974e1f69627832ef. 2024-11-24T22:15:33,499 ERROR [RS_FLUSH_OPERATIONS-regionserver/bd6592c37bb2:0-1 {event_type=RS_FLUSH_REGIONS, pid=102}] handler.RSProcedureHandler(58): pid=102 java.io.IOException: Unable to complete flush {ENCODED => 94f3ccf9f8a638fa974e1f69627832ef, NAME => 'TestAcidGuarantees,,1732486521610.94f3ccf9f8a638fa974e1f69627832ef.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T22:15:33,499 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bd6592c37bb2:0-1 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=102 java.io.IOException: Unable to complete flush {ENCODED => 94f3ccf9f8a638fa974e1f69627832ef, NAME => 'TestAcidGuarantees,,1732486521610.94f3ccf9f8a638fa974e1f69627832ef.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T22:15:33,500 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37701 {}] master.HMaster(4177): Remote procedure failed, pid=102 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 94f3ccf9f8a638fa974e1f69627832ef, NAME => 'TestAcidGuarantees,,1732486521610.94f3ccf9f8a638fa974e1f69627832ef.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 94f3ccf9f8a638fa974e1f69627832ef, NAME => 'TestAcidGuarantees,,1732486521610.94f3ccf9f8a638fa974e1f69627832ef.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T22:15:33,637 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40087 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94f3ccf9f8a638fa974e1f69627832ef, server=bd6592c37bb2,40087,1732486423713 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-24T22:15:33,637 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40087 {}] ipc.CallRunner(138): callId: 58 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:32822 deadline: 1732486593631, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94f3ccf9f8a638fa974e1f69627832ef, server=bd6592c37bb2,40087,1732486423713 2024-11-24T22:15:33,638 WARN [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRpcRetryingCaller(177): Call to bd6592c37bb2,40087,1732486423713 for 'test_row_1' in 94f3ccf9f8a638fa974e1f69627832ef of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2220 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94f3ccf9f8a638fa974e1f69627832ef, server=bd6592c37bb2,40087,1732486423713 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor44.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94f3ccf9f8a638fa974e1f69627832ef, server=bd6592c37bb2,40087,1732486423713 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-24T22:15:33,638 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732486521610.94f3ccf9f8a638fa974e1f69627832ef., hostname=bd6592c37bb2,40087,1732486423713, seqNum=5 , the old value is region=TestAcidGuarantees,,1732486521610.94f3ccf9f8a638fa974e1f69627832ef., hostname=bd6592c37bb2,40087,1732486423713, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94f3ccf9f8a638fa974e1f69627832ef, server=bd6592c37bb2,40087,1732486423713 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T22:15:33,638 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732486521610.94f3ccf9f8a638fa974e1f69627832ef., hostname=bd6592c37bb2,40087,1732486423713, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94f3ccf9f8a638fa974e1f69627832ef, server=bd6592c37bb2,40087,1732486423713 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T22:15:33,638 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732486521610.94f3ccf9f8a638fa974e1f69627832ef., hostname=bd6592c37bb2,40087,1732486423713, seqNum=5 because the exception is null or not the one we care about 2024-11-24T22:15:33,651 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=40087 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=102 2024-11-24T22:15:33,652 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bd6592c37bb2:0-2 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732486521610.94f3ccf9f8a638fa974e1f69627832ef. 2024-11-24T22:15:33,652 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bd6592c37bb2:0-2 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732486521610.94f3ccf9f8a638fa974e1f69627832ef. as already flushing 2024-11-24T22:15:33,652 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bd6592c37bb2:0-2 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732486521610.94f3ccf9f8a638fa974e1f69627832ef. 2024-11-24T22:15:33,652 ERROR [RS_FLUSH_OPERATIONS-regionserver/bd6592c37bb2:0-2 {event_type=RS_FLUSH_REGIONS, pid=102}] handler.RSProcedureHandler(58): pid=102 java.io.IOException: Unable to complete flush {ENCODED => 94f3ccf9f8a638fa974e1f69627832ef, NAME => 'TestAcidGuarantees,,1732486521610.94f3ccf9f8a638fa974e1f69627832ef.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T22:15:33,652 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bd6592c37bb2:0-2 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=102 java.io.IOException: Unable to complete flush {ENCODED => 94f3ccf9f8a638fa974e1f69627832ef, NAME => 'TestAcidGuarantees,,1732486521610.94f3ccf9f8a638fa974e1f69627832ef.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T22:15:33,653 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37701 {}] master.HMaster(4177): Remote procedure failed, pid=102 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 94f3ccf9f8a638fa974e1f69627832ef, NAME => 'TestAcidGuarantees,,1732486521610.94f3ccf9f8a638fa974e1f69627832ef.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 94f3ccf9f8a638fa974e1f69627832ef, NAME => 'TestAcidGuarantees,,1732486521610.94f3ccf9f8a638fa974e1f69627832ef.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T22:15:33,671 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37701 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=101 2024-11-24T22:15:33,677 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40087 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94f3ccf9f8a638fa974e1f69627832ef, server=bd6592c37bb2,40087,1732486423713 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-24T22:15:33,677 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40087 {}] ipc.CallRunner(138): callId: 48 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:32846 deadline: 1732486593671, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94f3ccf9f8a638fa974e1f69627832ef, server=bd6592c37bb2,40087,1732486423713 2024-11-24T22:15:33,677 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732486521610.94f3ccf9f8a638fa974e1f69627832ef., hostname=bd6592c37bb2,40087,1732486423713, seqNum=5 , the old value is region=TestAcidGuarantees,,1732486521610.94f3ccf9f8a638fa974e1f69627832ef., hostname=bd6592c37bb2,40087,1732486423713, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94f3ccf9f8a638fa974e1f69627832ef, server=bd6592c37bb2,40087,1732486423713 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T22:15:33,677 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732486521610.94f3ccf9f8a638fa974e1f69627832ef., hostname=bd6592c37bb2,40087,1732486423713, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94f3ccf9f8a638fa974e1f69627832ef, server=bd6592c37bb2,40087,1732486423713 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T22:15:33,677 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732486521610.94f3ccf9f8a638fa974e1f69627832ef., hostname=bd6592c37bb2,40087,1732486423713, seqNum=5 because the exception is null or not the one we care about 2024-11-24T22:15:33,679 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40087 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94f3ccf9f8a638fa974e1f69627832ef, server=bd6592c37bb2,40087,1732486423713 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-24T22:15:33,679 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40087 {}] ipc.CallRunner(138): callId: 34 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:32832 deadline: 1732486593671, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94f3ccf9f8a638fa974e1f69627832ef, server=bd6592c37bb2,40087,1732486423713 2024-11-24T22:15:33,679 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40087 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94f3ccf9f8a638fa974e1f69627832ef, server=bd6592c37bb2,40087,1732486423713 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-24T22:15:33,679 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732486521610.94f3ccf9f8a638fa974e1f69627832ef., hostname=bd6592c37bb2,40087,1732486423713, seqNum=5 , the old value is region=TestAcidGuarantees,,1732486521610.94f3ccf9f8a638fa974e1f69627832ef., hostname=bd6592c37bb2,40087,1732486423713, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94f3ccf9f8a638fa974e1f69627832ef, server=bd6592c37bb2,40087,1732486423713 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T22:15:33,679 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732486521610.94f3ccf9f8a638fa974e1f69627832ef., hostname=bd6592c37bb2,40087,1732486423713, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94f3ccf9f8a638fa974e1f69627832ef, server=bd6592c37bb2,40087,1732486423713 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T22:15:33,679 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40087 {}] ipc.CallRunner(138): callId: 51 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:32774 deadline: 1732486593671, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94f3ccf9f8a638fa974e1f69627832ef, server=bd6592c37bb2,40087,1732486423713 2024-11-24T22:15:33,679 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732486521610.94f3ccf9f8a638fa974e1f69627832ef., hostname=bd6592c37bb2,40087,1732486423713, seqNum=5 because the exception is null or not the one we care about 2024-11-24T22:15:33,680 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40087 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94f3ccf9f8a638fa974e1f69627832ef, server=bd6592c37bb2,40087,1732486423713 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-24T22:15:33,680 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40087 {}] ipc.CallRunner(138): callId: 51 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:32810 deadline: 1732486593671, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94f3ccf9f8a638fa974e1f69627832ef, server=bd6592c37bb2,40087,1732486423713 2024-11-24T22:15:33,680 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732486521610.94f3ccf9f8a638fa974e1f69627832ef., hostname=bd6592c37bb2,40087,1732486423713, seqNum=5 , the old value is region=TestAcidGuarantees,,1732486521610.94f3ccf9f8a638fa974e1f69627832ef., hostname=bd6592c37bb2,40087,1732486423713, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94f3ccf9f8a638fa974e1f69627832ef, server=bd6592c37bb2,40087,1732486423713 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T22:15:33,680 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732486521610.94f3ccf9f8a638fa974e1f69627832ef., hostname=bd6592c37bb2,40087,1732486423713, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94f3ccf9f8a638fa974e1f69627832ef, server=bd6592c37bb2,40087,1732486423713 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T22:15:33,680 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732486521610.94f3ccf9f8a638fa974e1f69627832ef., hostname=bd6592c37bb2,40087,1732486423713, seqNum=5 because the exception is null or not the one we care about 2024-11-24T22:15:33,680 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732486521610.94f3ccf9f8a638fa974e1f69627832ef., hostname=bd6592c37bb2,40087,1732486423713, seqNum=5 , the old value is region=TestAcidGuarantees,,1732486521610.94f3ccf9f8a638fa974e1f69627832ef., hostname=bd6592c37bb2,40087,1732486423713, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94f3ccf9f8a638fa974e1f69627832ef, server=bd6592c37bb2,40087,1732486423713 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T22:15:33,680 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732486521610.94f3ccf9f8a638fa974e1f69627832ef., hostname=bd6592c37bb2,40087,1732486423713, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94f3ccf9f8a638fa974e1f69627832ef, server=bd6592c37bb2,40087,1732486423713 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T22:15:33,680 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732486521610.94f3ccf9f8a638fa974e1f69627832ef., hostname=bd6592c37bb2,40087,1732486423713, seqNum=5 because the exception is null or not the one we care about 2024-11-24T22:15:33,769 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=35.78 KB at sequenceid=141 (bloomFilter=true), to=hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/94f3ccf9f8a638fa974e1f69627832ef/.tmp/C/a3395d0e4c1c43a98e2163a3d14415df 2024-11-24T22:15:33,772 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/94f3ccf9f8a638fa974e1f69627832ef/.tmp/A/697a1699f64d45fcb65780147612779b as hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/94f3ccf9f8a638fa974e1f69627832ef/A/697a1699f64d45fcb65780147612779b 2024-11-24T22:15:33,776 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/94f3ccf9f8a638fa974e1f69627832ef/A/697a1699f64d45fcb65780147612779b, entries=200, sequenceid=141, filesize=38.8 K 2024-11-24T22:15:33,776 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/94f3ccf9f8a638fa974e1f69627832ef/.tmp/B/d141ed51dbd0448f9084a2a5552ef064 as hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/94f3ccf9f8a638fa974e1f69627832ef/B/d141ed51dbd0448f9084a2a5552ef064 2024-11-24T22:15:33,780 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/94f3ccf9f8a638fa974e1f69627832ef/B/d141ed51dbd0448f9084a2a5552ef064, entries=150, sequenceid=141, filesize=11.9 K 2024-11-24T22:15:33,781 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/94f3ccf9f8a638fa974e1f69627832ef/.tmp/C/a3395d0e4c1c43a98e2163a3d14415df as hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/94f3ccf9f8a638fa974e1f69627832ef/C/a3395d0e4c1c43a98e2163a3d14415df 2024-11-24T22:15:33,784 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/94f3ccf9f8a638fa974e1f69627832ef/C/a3395d0e4c1c43a98e2163a3d14415df, entries=150, sequenceid=141, filesize=11.9 K 2024-11-24T22:15:33,784 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~107.34 KB/109920, heapSize ~319.45 KB/327120, currentSize=73.80 KB/75570 for 94f3ccf9f8a638fa974e1f69627832ef in 1342ms, sequenceid=141, compaction requested=false 2024-11-24T22:15:33,784 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 94f3ccf9f8a638fa974e1f69627832ef: 2024-11-24T22:15:33,804 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=40087 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=102 2024-11-24T22:15:33,805 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bd6592c37bb2:0-0 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732486521610.94f3ccf9f8a638fa974e1f69627832ef. 2024-11-24T22:15:33,805 INFO [RS_FLUSH_OPERATIONS-regionserver/bd6592c37bb2:0-0 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.HRegion(2902): Flushing 94f3ccf9f8a638fa974e1f69627832ef 3/3 column families, dataSize=73.80 KB heapSize=219.89 KB 2024-11-24T22:15:33,805 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bd6592c37bb2:0-0 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 94f3ccf9f8a638fa974e1f69627832ef, store=A 2024-11-24T22:15:33,805 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bd6592c37bb2:0-0 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-24T22:15:33,805 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bd6592c37bb2:0-0 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 94f3ccf9f8a638fa974e1f69627832ef, store=B 2024-11-24T22:15:33,805 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bd6592c37bb2:0-0 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-24T22:15:33,805 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bd6592c37bb2:0-0 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 94f3ccf9f8a638fa974e1f69627832ef, store=C 2024-11-24T22:15:33,805 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bd6592c37bb2:0-0 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-24T22:15:33,811 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bd6592c37bb2:0-0 {event_type=RS_FLUSH_REGIONS, pid=102}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024112433497985dd284519a218a410b01c5320_94f3ccf9f8a638fa974e1f69627832ef is 50, key is test_row_0/A:col10/1732486532456/Put/seqid=0 2024-11-24T22:15:33,815 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45929 is added to blk_1073742220_1396 (size=12304) 2024-11-24T22:15:34,181 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37701 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=101 2024-11-24T22:15:34,215 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bd6592c37bb2:0-0 {event_type=RS_FLUSH_REGIONS, pid=102}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T22:15:34,219 INFO [RS_FLUSH_OPERATIONS-regionserver/bd6592c37bb2:0-0 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024112433497985dd284519a218a410b01c5320_94f3ccf9f8a638fa974e1f69627832ef to hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024112433497985dd284519a218a410b01c5320_94f3ccf9f8a638fa974e1f69627832ef 2024-11-24T22:15:34,220 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bd6592c37bb2:0-0 {event_type=RS_FLUSH_REGIONS, pid=102}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/94f3ccf9f8a638fa974e1f69627832ef/.tmp/A/c3b3073483d94256bdd7bddae194779a, store: [table=TestAcidGuarantees family=A region=94f3ccf9f8a638fa974e1f69627832ef] 2024-11-24T22:15:34,221 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bd6592c37bb2:0-0 {event_type=RS_FLUSH_REGIONS, pid=102}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/94f3ccf9f8a638fa974e1f69627832ef/.tmp/A/c3b3073483d94256bdd7bddae194779a is 175, key is test_row_0/A:col10/1732486532456/Put/seqid=0 2024-11-24T22:15:34,224 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45929 is added to blk_1073742221_1397 (size=31105) 2024-11-24T22:15:34,625 INFO [RS_FLUSH_OPERATIONS-regionserver/bd6592c37bb2:0-0 {event_type=RS_FLUSH_REGIONS, pid=102}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=156, memsize=24.6 K, hasBloomFilter=true, into tmp file hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/94f3ccf9f8a638fa974e1f69627832ef/.tmp/A/c3b3073483d94256bdd7bddae194779a 2024-11-24T22:15:34,633 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bd6592c37bb2:0-0 {event_type=RS_FLUSH_REGIONS, pid=102}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/94f3ccf9f8a638fa974e1f69627832ef/.tmp/B/aed5d43a422c4c0a892806e5dc6d9ddb is 50, key is test_row_0/B:col10/1732486532456/Put/seqid=0 2024-11-24T22:15:34,639 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45929 is added to blk_1073742222_1398 (size=12151) 2024-11-24T22:15:34,694 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40087 {}] regionserver.HRegion(8855): Flush requested on 94f3ccf9f8a638fa974e1f69627832ef 2024-11-24T22:15:34,694 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732486521610.94f3ccf9f8a638fa974e1f69627832ef. as already flushing 2024-11-24T22:15:34,735 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40087 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94f3ccf9f8a638fa974e1f69627832ef, server=bd6592c37bb2,40087,1732486423713 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-24T22:15:34,736 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40087 {}] ipc.CallRunner(138): callId: 55 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:32810 deadline: 1732486594722, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94f3ccf9f8a638fa974e1f69627832ef, server=bd6592c37bb2,40087,1732486423713 2024-11-24T22:15:34,736 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40087 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94f3ccf9f8a638fa974e1f69627832ef, server=bd6592c37bb2,40087,1732486423713 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-24T22:15:34,736 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732486521610.94f3ccf9f8a638fa974e1f69627832ef., hostname=bd6592c37bb2,40087,1732486423713, seqNum=5 , the old value is region=TestAcidGuarantees,,1732486521610.94f3ccf9f8a638fa974e1f69627832ef., hostname=bd6592c37bb2,40087,1732486423713, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94f3ccf9f8a638fa974e1f69627832ef, server=bd6592c37bb2,40087,1732486423713 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T22:15:34,736 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732486521610.94f3ccf9f8a638fa974e1f69627832ef., hostname=bd6592c37bb2,40087,1732486423713, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94f3ccf9f8a638fa974e1f69627832ef, server=bd6592c37bb2,40087,1732486423713 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T22:15:34,736 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40087 {}] ipc.CallRunner(138): callId: 53 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:32846 deadline: 1732486594725, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94f3ccf9f8a638fa974e1f69627832ef, server=bd6592c37bb2,40087,1732486423713 2024-11-24T22:15:34,736 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732486521610.94f3ccf9f8a638fa974e1f69627832ef., hostname=bd6592c37bb2,40087,1732486423713, seqNum=5 because the exception is null or not the one we care about 2024-11-24T22:15:34,736 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732486521610.94f3ccf9f8a638fa974e1f69627832ef., hostname=bd6592c37bb2,40087,1732486423713, seqNum=5 , the old value is region=TestAcidGuarantees,,1732486521610.94f3ccf9f8a638fa974e1f69627832ef., hostname=bd6592c37bb2,40087,1732486423713, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94f3ccf9f8a638fa974e1f69627832ef, server=bd6592c37bb2,40087,1732486423713 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T22:15:34,736 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732486521610.94f3ccf9f8a638fa974e1f69627832ef., hostname=bd6592c37bb2,40087,1732486423713, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94f3ccf9f8a638fa974e1f69627832ef, server=bd6592c37bb2,40087,1732486423713 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T22:15:34,736 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732486521610.94f3ccf9f8a638fa974e1f69627832ef., hostname=bd6592c37bb2,40087,1732486423713, seqNum=5 because the exception is null or not the one we care about 2024-11-24T22:15:34,739 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40087 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94f3ccf9f8a638fa974e1f69627832ef, server=bd6592c37bb2,40087,1732486423713 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-24T22:15:34,740 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40087 {}] ipc.CallRunner(138): callId: 39 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:32832 deadline: 1732486594728, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94f3ccf9f8a638fa974e1f69627832ef, server=bd6592c37bb2,40087,1732486423713 2024-11-24T22:15:34,740 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732486521610.94f3ccf9f8a638fa974e1f69627832ef., hostname=bd6592c37bb2,40087,1732486423713, seqNum=5 , the old value is region=TestAcidGuarantees,,1732486521610.94f3ccf9f8a638fa974e1f69627832ef., hostname=bd6592c37bb2,40087,1732486423713, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94f3ccf9f8a638fa974e1f69627832ef, server=bd6592c37bb2,40087,1732486423713 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T22:15:34,740 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732486521610.94f3ccf9f8a638fa974e1f69627832ef., hostname=bd6592c37bb2,40087,1732486423713, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94f3ccf9f8a638fa974e1f69627832ef, server=bd6592c37bb2,40087,1732486423713 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T22:15:34,740 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732486521610.94f3ccf9f8a638fa974e1f69627832ef., hostname=bd6592c37bb2,40087,1732486423713, seqNum=5 because the exception is null or not the one we care about 2024-11-24T22:15:34,742 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40087 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94f3ccf9f8a638fa974e1f69627832ef, server=bd6592c37bb2,40087,1732486423713 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-24T22:15:34,743 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40087 {}] ipc.CallRunner(138): callId: 56 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:32774 deadline: 1732486594729, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94f3ccf9f8a638fa974e1f69627832ef, server=bd6592c37bb2,40087,1732486423713 2024-11-24T22:15:34,743 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732486521610.94f3ccf9f8a638fa974e1f69627832ef., hostname=bd6592c37bb2,40087,1732486423713, seqNum=5 , the old value is region=TestAcidGuarantees,,1732486521610.94f3ccf9f8a638fa974e1f69627832ef., hostname=bd6592c37bb2,40087,1732486423713, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94f3ccf9f8a638fa974e1f69627832ef, server=bd6592c37bb2,40087,1732486423713 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T22:15:34,743 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732486521610.94f3ccf9f8a638fa974e1f69627832ef., hostname=bd6592c37bb2,40087,1732486423713, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94f3ccf9f8a638fa974e1f69627832ef, server=bd6592c37bb2,40087,1732486423713 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T22:15:34,743 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732486521610.94f3ccf9f8a638fa974e1f69627832ef., hostname=bd6592c37bb2,40087,1732486423713, seqNum=5 because the exception is null or not the one we care about 2024-11-24T22:15:34,847 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40087 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94f3ccf9f8a638fa974e1f69627832ef, server=bd6592c37bb2,40087,1732486423713 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-24T22:15:34,847 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40087 {}] ipc.CallRunner(138): callId: 56 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:32810 deadline: 1732486594841, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94f3ccf9f8a638fa974e1f69627832ef, server=bd6592c37bb2,40087,1732486423713 2024-11-24T22:15:34,847 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40087 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94f3ccf9f8a638fa974e1f69627832ef, server=bd6592c37bb2,40087,1732486423713 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-24T22:15:34,847 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732486521610.94f3ccf9f8a638fa974e1f69627832ef., hostname=bd6592c37bb2,40087,1732486423713, seqNum=5 , the old value is region=TestAcidGuarantees,,1732486521610.94f3ccf9f8a638fa974e1f69627832ef., hostname=bd6592c37bb2,40087,1732486423713, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94f3ccf9f8a638fa974e1f69627832ef, server=bd6592c37bb2,40087,1732486423713 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T22:15:34,847 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40087 {}] ipc.CallRunner(138): callId: 54 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:32846 deadline: 1732486594841, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94f3ccf9f8a638fa974e1f69627832ef, server=bd6592c37bb2,40087,1732486423713 2024-11-24T22:15:34,847 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732486521610.94f3ccf9f8a638fa974e1f69627832ef., hostname=bd6592c37bb2,40087,1732486423713, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94f3ccf9f8a638fa974e1f69627832ef, server=bd6592c37bb2,40087,1732486423713 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T22:15:34,847 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732486521610.94f3ccf9f8a638fa974e1f69627832ef., hostname=bd6592c37bb2,40087,1732486423713, seqNum=5 because the exception is null or not the one we care about 2024-11-24T22:15:34,848 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732486521610.94f3ccf9f8a638fa974e1f69627832ef., hostname=bd6592c37bb2,40087,1732486423713, seqNum=5 , the old value is region=TestAcidGuarantees,,1732486521610.94f3ccf9f8a638fa974e1f69627832ef., hostname=bd6592c37bb2,40087,1732486423713, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94f3ccf9f8a638fa974e1f69627832ef, server=bd6592c37bb2,40087,1732486423713 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T22:15:34,848 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732486521610.94f3ccf9f8a638fa974e1f69627832ef., hostname=bd6592c37bb2,40087,1732486423713, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94f3ccf9f8a638fa974e1f69627832ef, server=bd6592c37bb2,40087,1732486423713 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T22:15:34,848 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732486521610.94f3ccf9f8a638fa974e1f69627832ef., hostname=bd6592c37bb2,40087,1732486423713, seqNum=5 because the exception is null or not the one we care about 2024-11-24T22:15:34,861 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40087 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94f3ccf9f8a638fa974e1f69627832ef, server=bd6592c37bb2,40087,1732486423713 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-24T22:15:34,862 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40087 {}] ipc.CallRunner(138): callId: 40 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:32832 deadline: 1732486594851, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94f3ccf9f8a638fa974e1f69627832ef, server=bd6592c37bb2,40087,1732486423713 2024-11-24T22:15:34,862 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732486521610.94f3ccf9f8a638fa974e1f69627832ef., hostname=bd6592c37bb2,40087,1732486423713, seqNum=5 , the old value is region=TestAcidGuarantees,,1732486521610.94f3ccf9f8a638fa974e1f69627832ef., hostname=bd6592c37bb2,40087,1732486423713, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94f3ccf9f8a638fa974e1f69627832ef, server=bd6592c37bb2,40087,1732486423713 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T22:15:34,862 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40087 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94f3ccf9f8a638fa974e1f69627832ef, server=bd6592c37bb2,40087,1732486423713 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-24T22:15:34,862 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732486521610.94f3ccf9f8a638fa974e1f69627832ef., hostname=bd6592c37bb2,40087,1732486423713, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94f3ccf9f8a638fa974e1f69627832ef, server=bd6592c37bb2,40087,1732486423713 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T22:15:34,862 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732486521610.94f3ccf9f8a638fa974e1f69627832ef., hostname=bd6592c37bb2,40087,1732486423713, seqNum=5 because the exception is null or not the one we care about 2024-11-24T22:15:34,862 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40087 {}] ipc.CallRunner(138): callId: 57 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:32774 deadline: 1732486594851, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94f3ccf9f8a638fa974e1f69627832ef, server=bd6592c37bb2,40087,1732486423713 2024-11-24T22:15:34,862 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732486521610.94f3ccf9f8a638fa974e1f69627832ef., hostname=bd6592c37bb2,40087,1732486423713, seqNum=5 , the old value is region=TestAcidGuarantees,,1732486521610.94f3ccf9f8a638fa974e1f69627832ef., hostname=bd6592c37bb2,40087,1732486423713, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94f3ccf9f8a638fa974e1f69627832ef, server=bd6592c37bb2,40087,1732486423713 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T22:15:34,862 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732486521610.94f3ccf9f8a638fa974e1f69627832ef., hostname=bd6592c37bb2,40087,1732486423713, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94f3ccf9f8a638fa974e1f69627832ef, server=bd6592c37bb2,40087,1732486423713 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T22:15:34,862 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732486521610.94f3ccf9f8a638fa974e1f69627832ef., hostname=bd6592c37bb2,40087,1732486423713, seqNum=5 because the exception is null or not the one we care about 2024-11-24T22:15:35,040 INFO [RS_FLUSH_OPERATIONS-regionserver/bd6592c37bb2:0-0 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=24.60 KB at sequenceid=156 (bloomFilter=true), to=hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/94f3ccf9f8a638fa974e1f69627832ef/.tmp/B/aed5d43a422c4c0a892806e5dc6d9ddb 2024-11-24T22:15:35,047 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bd6592c37bb2:0-0 {event_type=RS_FLUSH_REGIONS, pid=102}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/94f3ccf9f8a638fa974e1f69627832ef/.tmp/C/a12ca2c795164aae8da91298a1edd817 is 50, key is test_row_0/C:col10/1732486532456/Put/seqid=0 2024-11-24T22:15:35,050 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45929 is added to blk_1073742223_1399 (size=12151) 2024-11-24T22:15:35,054 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40087 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94f3ccf9f8a638fa974e1f69627832ef, server=bd6592c37bb2,40087,1732486423713 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-24T22:15:35,054 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40087 {}] ipc.CallRunner(138): callId: 57 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:32810 deadline: 1732486595051, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94f3ccf9f8a638fa974e1f69627832ef, server=bd6592c37bb2,40087,1732486423713 2024-11-24T22:15:35,054 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40087 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94f3ccf9f8a638fa974e1f69627832ef, server=bd6592c37bb2,40087,1732486423713 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-24T22:15:35,054 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732486521610.94f3ccf9f8a638fa974e1f69627832ef., hostname=bd6592c37bb2,40087,1732486423713, seqNum=5 , the old value is region=TestAcidGuarantees,,1732486521610.94f3ccf9f8a638fa974e1f69627832ef., hostname=bd6592c37bb2,40087,1732486423713, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94f3ccf9f8a638fa974e1f69627832ef, server=bd6592c37bb2,40087,1732486423713 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T22:15:35,054 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732486521610.94f3ccf9f8a638fa974e1f69627832ef., hostname=bd6592c37bb2,40087,1732486423713, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94f3ccf9f8a638fa974e1f69627832ef, server=bd6592c37bb2,40087,1732486423713 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T22:15:35,054 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40087 {}] ipc.CallRunner(138): callId: 55 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:32846 deadline: 1732486595051, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94f3ccf9f8a638fa974e1f69627832ef, server=bd6592c37bb2,40087,1732486423713 2024-11-24T22:15:35,054 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732486521610.94f3ccf9f8a638fa974e1f69627832ef., hostname=bd6592c37bb2,40087,1732486423713, seqNum=5 because the exception is null or not the one we care about 2024-11-24T22:15:35,054 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732486521610.94f3ccf9f8a638fa974e1f69627832ef., hostname=bd6592c37bb2,40087,1732486423713, seqNum=5 , the old value is region=TestAcidGuarantees,,1732486521610.94f3ccf9f8a638fa974e1f69627832ef., hostname=bd6592c37bb2,40087,1732486423713, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94f3ccf9f8a638fa974e1f69627832ef, server=bd6592c37bb2,40087,1732486423713 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T22:15:35,054 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732486521610.94f3ccf9f8a638fa974e1f69627832ef., hostname=bd6592c37bb2,40087,1732486423713, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94f3ccf9f8a638fa974e1f69627832ef, server=bd6592c37bb2,40087,1732486423713 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T22:15:35,054 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732486521610.94f3ccf9f8a638fa974e1f69627832ef., hostname=bd6592c37bb2,40087,1732486423713, seqNum=5 because the exception is null or not the one we care about 2024-11-24T22:15:35,080 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40087 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94f3ccf9f8a638fa974e1f69627832ef, server=bd6592c37bb2,40087,1732486423713 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-24T22:15:35,081 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40087 {}] ipc.CallRunner(138): callId: 41 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:32832 deadline: 1732486595071, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94f3ccf9f8a638fa974e1f69627832ef, server=bd6592c37bb2,40087,1732486423713 2024-11-24T22:15:35,081 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40087 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94f3ccf9f8a638fa974e1f69627832ef, server=bd6592c37bb2,40087,1732486423713 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-24T22:15:35,081 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732486521610.94f3ccf9f8a638fa974e1f69627832ef., hostname=bd6592c37bb2,40087,1732486423713, seqNum=5 , the old value is region=TestAcidGuarantees,,1732486521610.94f3ccf9f8a638fa974e1f69627832ef., hostname=bd6592c37bb2,40087,1732486423713, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94f3ccf9f8a638fa974e1f69627832ef, server=bd6592c37bb2,40087,1732486423713 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T22:15:35,081 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732486521610.94f3ccf9f8a638fa974e1f69627832ef., hostname=bd6592c37bb2,40087,1732486423713, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94f3ccf9f8a638fa974e1f69627832ef, server=bd6592c37bb2,40087,1732486423713 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T22:15:35,081 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40087 {}] ipc.CallRunner(138): callId: 58 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:32774 deadline: 1732486595071, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94f3ccf9f8a638fa974e1f69627832ef, server=bd6592c37bb2,40087,1732486423713 2024-11-24T22:15:35,081 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732486521610.94f3ccf9f8a638fa974e1f69627832ef., hostname=bd6592c37bb2,40087,1732486423713, seqNum=5 because the exception is null or not the one we care about 2024-11-24T22:15:35,081 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732486521610.94f3ccf9f8a638fa974e1f69627832ef., hostname=bd6592c37bb2,40087,1732486423713, seqNum=5 , the old value is region=TestAcidGuarantees,,1732486521610.94f3ccf9f8a638fa974e1f69627832ef., hostname=bd6592c37bb2,40087,1732486423713, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94f3ccf9f8a638fa974e1f69627832ef, server=bd6592c37bb2,40087,1732486423713 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T22:15:35,081 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732486521610.94f3ccf9f8a638fa974e1f69627832ef., hostname=bd6592c37bb2,40087,1732486423713, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94f3ccf9f8a638fa974e1f69627832ef, server=bd6592c37bb2,40087,1732486423713 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T22:15:35,081 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732486521610.94f3ccf9f8a638fa974e1f69627832ef., hostname=bd6592c37bb2,40087,1732486423713, seqNum=5 because the exception is null or not the one we care about 2024-11-24T22:15:35,191 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37701 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=101 2024-11-24T22:15:35,368 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40087 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94f3ccf9f8a638fa974e1f69627832ef, server=bd6592c37bb2,40087,1732486423713 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-24T22:15:35,369 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40087 {}] ipc.CallRunner(138): callId: 58 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:32810 deadline: 1732486595360, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94f3ccf9f8a638fa974e1f69627832ef, server=bd6592c37bb2,40087,1732486423713 2024-11-24T22:15:35,369 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40087 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94f3ccf9f8a638fa974e1f69627832ef, server=bd6592c37bb2,40087,1732486423713 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-24T22:15:35,369 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732486521610.94f3ccf9f8a638fa974e1f69627832ef., hostname=bd6592c37bb2,40087,1732486423713, seqNum=5 , the old value is region=TestAcidGuarantees,,1732486521610.94f3ccf9f8a638fa974e1f69627832ef., hostname=bd6592c37bb2,40087,1732486423713, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94f3ccf9f8a638fa974e1f69627832ef, server=bd6592c37bb2,40087,1732486423713 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T22:15:35,369 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732486521610.94f3ccf9f8a638fa974e1f69627832ef., hostname=bd6592c37bb2,40087,1732486423713, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94f3ccf9f8a638fa974e1f69627832ef, server=bd6592c37bb2,40087,1732486423713 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T22:15:35,369 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40087 {}] ipc.CallRunner(138): callId: 56 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:32846 deadline: 1732486595360, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94f3ccf9f8a638fa974e1f69627832ef, server=bd6592c37bb2,40087,1732486423713 2024-11-24T22:15:35,369 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732486521610.94f3ccf9f8a638fa974e1f69627832ef., hostname=bd6592c37bb2,40087,1732486423713, seqNum=5 because the exception is null or not the one we care about 2024-11-24T22:15:35,369 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732486521610.94f3ccf9f8a638fa974e1f69627832ef., hostname=bd6592c37bb2,40087,1732486423713, seqNum=5 , the old value is region=TestAcidGuarantees,,1732486521610.94f3ccf9f8a638fa974e1f69627832ef., hostname=bd6592c37bb2,40087,1732486423713, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94f3ccf9f8a638fa974e1f69627832ef, server=bd6592c37bb2,40087,1732486423713 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T22:15:35,369 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732486521610.94f3ccf9f8a638fa974e1f69627832ef., hostname=bd6592c37bb2,40087,1732486423713, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94f3ccf9f8a638fa974e1f69627832ef, server=bd6592c37bb2,40087,1732486423713 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T22:15:35,369 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732486521610.94f3ccf9f8a638fa974e1f69627832ef., hostname=bd6592c37bb2,40087,1732486423713, seqNum=5 because the exception is null or not the one we care about 2024-11-24T22:15:35,401 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40087 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94f3ccf9f8a638fa974e1f69627832ef, server=bd6592c37bb2,40087,1732486423713 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-24T22:15:35,401 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40087 {}] ipc.CallRunner(138): callId: 42 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:32832 deadline: 1732486595391, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94f3ccf9f8a638fa974e1f69627832ef, server=bd6592c37bb2,40087,1732486423713 2024-11-24T22:15:35,402 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732486521610.94f3ccf9f8a638fa974e1f69627832ef., hostname=bd6592c37bb2,40087,1732486423713, seqNum=5 , the old value is region=TestAcidGuarantees,,1732486521610.94f3ccf9f8a638fa974e1f69627832ef., hostname=bd6592c37bb2,40087,1732486423713, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94f3ccf9f8a638fa974e1f69627832ef, server=bd6592c37bb2,40087,1732486423713 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T22:15:35,402 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40087 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94f3ccf9f8a638fa974e1f69627832ef, server=bd6592c37bb2,40087,1732486423713 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-24T22:15:35,402 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732486521610.94f3ccf9f8a638fa974e1f69627832ef., hostname=bd6592c37bb2,40087,1732486423713, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94f3ccf9f8a638fa974e1f69627832ef, server=bd6592c37bb2,40087,1732486423713 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T22:15:35,402 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732486521610.94f3ccf9f8a638fa974e1f69627832ef., hostname=bd6592c37bb2,40087,1732486423713, seqNum=5 because the exception is null or not the one we care about 2024-11-24T22:15:35,402 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40087 {}] ipc.CallRunner(138): callId: 59 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:32774 deadline: 1732486595391, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94f3ccf9f8a638fa974e1f69627832ef, server=bd6592c37bb2,40087,1732486423713 2024-11-24T22:15:35,402 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732486521610.94f3ccf9f8a638fa974e1f69627832ef., hostname=bd6592c37bb2,40087,1732486423713, seqNum=5 , the old value is region=TestAcidGuarantees,,1732486521610.94f3ccf9f8a638fa974e1f69627832ef., hostname=bd6592c37bb2,40087,1732486423713, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94f3ccf9f8a638fa974e1f69627832ef, server=bd6592c37bb2,40087,1732486423713 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T22:15:35,402 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732486521610.94f3ccf9f8a638fa974e1f69627832ef., hostname=bd6592c37bb2,40087,1732486423713, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94f3ccf9f8a638fa974e1f69627832ef, server=bd6592c37bb2,40087,1732486423713 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T22:15:35,402 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732486521610.94f3ccf9f8a638fa974e1f69627832ef., hostname=bd6592c37bb2,40087,1732486423713, seqNum=5 because the exception is null or not the one we care about 2024-11-24T22:15:35,451 INFO [RS_FLUSH_OPERATIONS-regionserver/bd6592c37bb2:0-0 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=24.60 KB at sequenceid=156 (bloomFilter=true), to=hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/94f3ccf9f8a638fa974e1f69627832ef/.tmp/C/a12ca2c795164aae8da91298a1edd817 2024-11-24T22:15:35,455 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bd6592c37bb2:0-0 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/94f3ccf9f8a638fa974e1f69627832ef/.tmp/A/c3b3073483d94256bdd7bddae194779a as hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/94f3ccf9f8a638fa974e1f69627832ef/A/c3b3073483d94256bdd7bddae194779a 2024-11-24T22:15:35,459 INFO [RS_FLUSH_OPERATIONS-regionserver/bd6592c37bb2:0-0 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/94f3ccf9f8a638fa974e1f69627832ef/A/c3b3073483d94256bdd7bddae194779a, entries=150, sequenceid=156, filesize=30.4 K 2024-11-24T22:15:35,459 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bd6592c37bb2:0-0 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/94f3ccf9f8a638fa974e1f69627832ef/.tmp/B/aed5d43a422c4c0a892806e5dc6d9ddb as hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/94f3ccf9f8a638fa974e1f69627832ef/B/aed5d43a422c4c0a892806e5dc6d9ddb 2024-11-24T22:15:35,463 INFO [RS_FLUSH_OPERATIONS-regionserver/bd6592c37bb2:0-0 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/94f3ccf9f8a638fa974e1f69627832ef/B/aed5d43a422c4c0a892806e5dc6d9ddb, entries=150, sequenceid=156, filesize=11.9 K 2024-11-24T22:15:35,463 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bd6592c37bb2:0-0 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/94f3ccf9f8a638fa974e1f69627832ef/.tmp/C/a12ca2c795164aae8da91298a1edd817 as hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/94f3ccf9f8a638fa974e1f69627832ef/C/a12ca2c795164aae8da91298a1edd817 2024-11-24T22:15:35,466 INFO [RS_FLUSH_OPERATIONS-regionserver/bd6592c37bb2:0-0 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/94f3ccf9f8a638fa974e1f69627832ef/C/a12ca2c795164aae8da91298a1edd817, entries=150, sequenceid=156, filesize=11.9 K 2024-11-24T22:15:35,467 INFO [RS_FLUSH_OPERATIONS-regionserver/bd6592c37bb2:0-0 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.HRegion(3140): Finished flush of dataSize ~73.80 KB/75570, heapSize ~219.84 KB/225120, currentSize=100.63 KB/103050 for 94f3ccf9f8a638fa974e1f69627832ef in 1662ms, sequenceid=156, compaction requested=true 2024-11-24T22:15:35,467 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bd6592c37bb2:0-0 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.HRegion(2603): Flush status journal for 94f3ccf9f8a638fa974e1f69627832ef: 2024-11-24T22:15:35,467 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bd6592c37bb2:0-0 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732486521610.94f3ccf9f8a638fa974e1f69627832ef. 2024-11-24T22:15:35,467 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bd6592c37bb2:0-0 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=102 2024-11-24T22:15:35,467 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37701 {}] master.HMaster(4169): Remote procedure done, pid=102 2024-11-24T22:15:35,470 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=102, resume processing ppid=101 2024-11-24T22:15:35,470 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=102, ppid=101, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 2.4270 sec 2024-11-24T22:15:35,472 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=101, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=101, table=TestAcidGuarantees in 2.4330 sec 2024-11-24T22:15:35,657 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40087 {}] regionserver.HRegion(8855): Flush requested on 94f3ccf9f8a638fa974e1f69627832ef 2024-11-24T22:15:35,658 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 94f3ccf9f8a638fa974e1f69627832ef 3/3 column families, dataSize=107.34 KB heapSize=319.50 KB 2024-11-24T22:15:35,658 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 94f3ccf9f8a638fa974e1f69627832ef, store=A 2024-11-24T22:15:35,658 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-24T22:15:35,658 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 94f3ccf9f8a638fa974e1f69627832ef, store=B 2024-11-24T22:15:35,658 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-24T22:15:35,658 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 94f3ccf9f8a638fa974e1f69627832ef, store=C 2024-11-24T22:15:35,658 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-24T22:15:35,665 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241124ba52deec5a5a4309abd261c1086afe25_94f3ccf9f8a638fa974e1f69627832ef is 50, key is test_row_0/A:col10/1732486534720/Put/seqid=0 2024-11-24T22:15:35,668 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45929 is added to blk_1073742224_1400 (size=14794) 2024-11-24T22:15:35,749 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40087 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94f3ccf9f8a638fa974e1f69627832ef, server=bd6592c37bb2,40087,1732486423713 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-24T22:15:35,749 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40087 {}] ipc.CallRunner(138): callId: 70 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:32822 deadline: 1732486595741, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94f3ccf9f8a638fa974e1f69627832ef, server=bd6592c37bb2,40087,1732486423713 2024-11-24T22:15:35,749 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732486521610.94f3ccf9f8a638fa974e1f69627832ef., hostname=bd6592c37bb2,40087,1732486423713, seqNum=5 , the old value is region=TestAcidGuarantees,,1732486521610.94f3ccf9f8a638fa974e1f69627832ef., hostname=bd6592c37bb2,40087,1732486423713, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94f3ccf9f8a638fa974e1f69627832ef, server=bd6592c37bb2,40087,1732486423713 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T22:15:35,749 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732486521610.94f3ccf9f8a638fa974e1f69627832ef., hostname=bd6592c37bb2,40087,1732486423713, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94f3ccf9f8a638fa974e1f69627832ef, server=bd6592c37bb2,40087,1732486423713 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T22:15:35,749 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732486521610.94f3ccf9f8a638fa974e1f69627832ef., hostname=bd6592c37bb2,40087,1732486423713, seqNum=5 because the exception is null or not the one we care about 2024-11-24T22:15:35,856 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40087 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94f3ccf9f8a638fa974e1f69627832ef, server=bd6592c37bb2,40087,1732486423713 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-24T22:15:35,856 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40087 {}] ipc.CallRunner(138): callId: 71 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:32822 deadline: 1732486595850, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94f3ccf9f8a638fa974e1f69627832ef, server=bd6592c37bb2,40087,1732486423713 2024-11-24T22:15:35,857 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732486521610.94f3ccf9f8a638fa974e1f69627832ef., hostname=bd6592c37bb2,40087,1732486423713, seqNum=5 , the old value is region=TestAcidGuarantees,,1732486521610.94f3ccf9f8a638fa974e1f69627832ef., hostname=bd6592c37bb2,40087,1732486423713, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94f3ccf9f8a638fa974e1f69627832ef, server=bd6592c37bb2,40087,1732486423713 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T22:15:35,857 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732486521610.94f3ccf9f8a638fa974e1f69627832ef., hostname=bd6592c37bb2,40087,1732486423713, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94f3ccf9f8a638fa974e1f69627832ef, server=bd6592c37bb2,40087,1732486423713 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T22:15:35,857 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732486521610.94f3ccf9f8a638fa974e1f69627832ef., hostname=bd6592c37bb2,40087,1732486423713, seqNum=5 because the exception is null or not the one we care about 2024-11-24T22:15:35,890 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40087 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94f3ccf9f8a638fa974e1f69627832ef, server=bd6592c37bb2,40087,1732486423713 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-24T22:15:35,890 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40087 {}] ipc.CallRunner(138): callId: 59 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:32810 deadline: 1732486595881, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94f3ccf9f8a638fa974e1f69627832ef, server=bd6592c37bb2,40087,1732486423713 2024-11-24T22:15:35,890 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40087 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94f3ccf9f8a638fa974e1f69627832ef, server=bd6592c37bb2,40087,1732486423713 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-24T22:15:35,890 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732486521610.94f3ccf9f8a638fa974e1f69627832ef., hostname=bd6592c37bb2,40087,1732486423713, seqNum=5 , the old value is region=TestAcidGuarantees,,1732486521610.94f3ccf9f8a638fa974e1f69627832ef., hostname=bd6592c37bb2,40087,1732486423713, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94f3ccf9f8a638fa974e1f69627832ef, server=bd6592c37bb2,40087,1732486423713 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T22:15:35,890 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40087 {}] ipc.CallRunner(138): callId: 57 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:32846 deadline: 1732486595881, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94f3ccf9f8a638fa974e1f69627832ef, server=bd6592c37bb2,40087,1732486423713 2024-11-24T22:15:35,890 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732486521610.94f3ccf9f8a638fa974e1f69627832ef., hostname=bd6592c37bb2,40087,1732486423713, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94f3ccf9f8a638fa974e1f69627832ef, server=bd6592c37bb2,40087,1732486423713 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T22:15:35,890 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732486521610.94f3ccf9f8a638fa974e1f69627832ef., hostname=bd6592c37bb2,40087,1732486423713, seqNum=5 because the exception is null or not the one we care about 2024-11-24T22:15:35,891 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732486521610.94f3ccf9f8a638fa974e1f69627832ef., hostname=bd6592c37bb2,40087,1732486423713, seqNum=5 , the old value is region=TestAcidGuarantees,,1732486521610.94f3ccf9f8a638fa974e1f69627832ef., hostname=bd6592c37bb2,40087,1732486423713, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94f3ccf9f8a638fa974e1f69627832ef, server=bd6592c37bb2,40087,1732486423713 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T22:15:35,891 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732486521610.94f3ccf9f8a638fa974e1f69627832ef., hostname=bd6592c37bb2,40087,1732486423713, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94f3ccf9f8a638fa974e1f69627832ef, server=bd6592c37bb2,40087,1732486423713 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T22:15:35,891 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732486521610.94f3ccf9f8a638fa974e1f69627832ef., hostname=bd6592c37bb2,40087,1732486423713, seqNum=5 because the exception is null or not the one we care about 2024-11-24T22:15:35,917 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40087 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94f3ccf9f8a638fa974e1f69627832ef, server=bd6592c37bb2,40087,1732486423713 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-24T22:15:35,917 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40087 {}] ipc.CallRunner(138): callId: 43 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:32832 deadline: 1732486595911, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94f3ccf9f8a638fa974e1f69627832ef, server=bd6592c37bb2,40087,1732486423713 2024-11-24T22:15:35,917 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732486521610.94f3ccf9f8a638fa974e1f69627832ef., hostname=bd6592c37bb2,40087,1732486423713, seqNum=5 , the old value is region=TestAcidGuarantees,,1732486521610.94f3ccf9f8a638fa974e1f69627832ef., hostname=bd6592c37bb2,40087,1732486423713, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94f3ccf9f8a638fa974e1f69627832ef, server=bd6592c37bb2,40087,1732486423713 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T22:15:35,917 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732486521610.94f3ccf9f8a638fa974e1f69627832ef., hostname=bd6592c37bb2,40087,1732486423713, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94f3ccf9f8a638fa974e1f69627832ef, server=bd6592c37bb2,40087,1732486423713 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T22:15:35,917 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40087 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94f3ccf9f8a638fa974e1f69627832ef, server=bd6592c37bb2,40087,1732486423713 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-24T22:15:35,917 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732486521610.94f3ccf9f8a638fa974e1f69627832ef., hostname=bd6592c37bb2,40087,1732486423713, seqNum=5 because the exception is null or not the one we care about 2024-11-24T22:15:35,917 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40087 {}] ipc.CallRunner(138): callId: 60 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:32774 deadline: 1732486595911, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94f3ccf9f8a638fa974e1f69627832ef, server=bd6592c37bb2,40087,1732486423713 2024-11-24T22:15:35,918 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732486521610.94f3ccf9f8a638fa974e1f69627832ef., hostname=bd6592c37bb2,40087,1732486423713, seqNum=5 , the old value is region=TestAcidGuarantees,,1732486521610.94f3ccf9f8a638fa974e1f69627832ef., hostname=bd6592c37bb2,40087,1732486423713, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94f3ccf9f8a638fa974e1f69627832ef, server=bd6592c37bb2,40087,1732486423713 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T22:15:35,918 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732486521610.94f3ccf9f8a638fa974e1f69627832ef., hostname=bd6592c37bb2,40087,1732486423713, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94f3ccf9f8a638fa974e1f69627832ef, server=bd6592c37bb2,40087,1732486423713 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T22:15:35,918 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732486521610.94f3ccf9f8a638fa974e1f69627832ef., hostname=bd6592c37bb2,40087,1732486423713, seqNum=5 because the exception is null or not the one we care about 2024-11-24T22:15:36,063 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40087 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94f3ccf9f8a638fa974e1f69627832ef, server=bd6592c37bb2,40087,1732486423713 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-24T22:15:36,063 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40087 {}] ipc.CallRunner(138): callId: 72 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:32822 deadline: 1732486596060, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94f3ccf9f8a638fa974e1f69627832ef, server=bd6592c37bb2,40087,1732486423713 2024-11-24T22:15:36,063 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732486521610.94f3ccf9f8a638fa974e1f69627832ef., hostname=bd6592c37bb2,40087,1732486423713, seqNum=5 , the old value is region=TestAcidGuarantees,,1732486521610.94f3ccf9f8a638fa974e1f69627832ef., hostname=bd6592c37bb2,40087,1732486423713, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94f3ccf9f8a638fa974e1f69627832ef, server=bd6592c37bb2,40087,1732486423713 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T22:15:36,063 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732486521610.94f3ccf9f8a638fa974e1f69627832ef., hostname=bd6592c37bb2,40087,1732486423713, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94f3ccf9f8a638fa974e1f69627832ef, server=bd6592c37bb2,40087,1732486423713 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T22:15:36,063 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732486521610.94f3ccf9f8a638fa974e1f69627832ef., hostname=bd6592c37bb2,40087,1732486423713, seqNum=5 because the exception is null or not the one we care about 2024-11-24T22:15:36,069 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T22:15:36,072 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241124ba52deec5a5a4309abd261c1086afe25_94f3ccf9f8a638fa974e1f69627832ef to hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241124ba52deec5a5a4309abd261c1086afe25_94f3ccf9f8a638fa974e1f69627832ef 2024-11-24T22:15:36,073 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/94f3ccf9f8a638fa974e1f69627832ef/.tmp/A/e25b282932044a15acb1a40c17957984, store: [table=TestAcidGuarantees family=A region=94f3ccf9f8a638fa974e1f69627832ef] 2024-11-24T22:15:36,074 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/94f3ccf9f8a638fa974e1f69627832ef/.tmp/A/e25b282932044a15acb1a40c17957984 is 175, key is test_row_0/A:col10/1732486534720/Put/seqid=0 2024-11-24T22:15:36,077 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45929 is added to blk_1073742225_1401 (size=39749) 2024-11-24T22:15:36,077 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=175, memsize=35.8 K, hasBloomFilter=true, into tmp file hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/94f3ccf9f8a638fa974e1f69627832ef/.tmp/A/e25b282932044a15acb1a40c17957984 2024-11-24T22:15:36,084 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/94f3ccf9f8a638fa974e1f69627832ef/.tmp/B/853e51745e104112b072f41f1a4b6852 is 50, key is test_row_0/B:col10/1732486534720/Put/seqid=0 2024-11-24T22:15:36,087 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45929 is added to blk_1073742226_1402 (size=12151) 2024-11-24T22:15:36,377 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40087 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94f3ccf9f8a638fa974e1f69627832ef, server=bd6592c37bb2,40087,1732486423713 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-24T22:15:36,377 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40087 {}] ipc.CallRunner(138): callId: 73 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:32822 deadline: 1732486596371, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94f3ccf9f8a638fa974e1f69627832ef, server=bd6592c37bb2,40087,1732486423713 2024-11-24T22:15:36,377 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732486521610.94f3ccf9f8a638fa974e1f69627832ef., hostname=bd6592c37bb2,40087,1732486423713, seqNum=5 , the old value is region=TestAcidGuarantees,,1732486521610.94f3ccf9f8a638fa974e1f69627832ef., hostname=bd6592c37bb2,40087,1732486423713, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94f3ccf9f8a638fa974e1f69627832ef, server=bd6592c37bb2,40087,1732486423713 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T22:15:36,377 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732486521610.94f3ccf9f8a638fa974e1f69627832ef., hostname=bd6592c37bb2,40087,1732486423713, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94f3ccf9f8a638fa974e1f69627832ef, server=bd6592c37bb2,40087,1732486423713 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T22:15:36,377 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732486521610.94f3ccf9f8a638fa974e1f69627832ef., hostname=bd6592c37bb2,40087,1732486423713, seqNum=5 because the exception is null or not the one we care about 2024-11-24T22:15:36,487 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=35.78 KB at sequenceid=175 (bloomFilter=true), to=hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/94f3ccf9f8a638fa974e1f69627832ef/.tmp/B/853e51745e104112b072f41f1a4b6852 2024-11-24T22:15:36,494 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/94f3ccf9f8a638fa974e1f69627832ef/.tmp/C/c68109f6bef04e81a21e61cdb622d3f0 is 50, key is test_row_0/C:col10/1732486534720/Put/seqid=0 2024-11-24T22:15:36,498 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45929 is added to blk_1073742227_1403 (size=12151) 2024-11-24T22:15:36,898 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40087 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94f3ccf9f8a638fa974e1f69627832ef, server=bd6592c37bb2,40087,1732486423713 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-24T22:15:36,898 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40087 {}] ipc.CallRunner(138): callId: 74 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:32822 deadline: 1732486596890, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94f3ccf9f8a638fa974e1f69627832ef, server=bd6592c37bb2,40087,1732486423713 2024-11-24T22:15:36,899 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732486521610.94f3ccf9f8a638fa974e1f69627832ef., hostname=bd6592c37bb2,40087,1732486423713, seqNum=5 , the old value is region=TestAcidGuarantees,,1732486521610.94f3ccf9f8a638fa974e1f69627832ef., hostname=bd6592c37bb2,40087,1732486423713, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94f3ccf9f8a638fa974e1f69627832ef, server=bd6592c37bb2,40087,1732486423713 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T22:15:36,899 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732486521610.94f3ccf9f8a638fa974e1f69627832ef., hostname=bd6592c37bb2,40087,1732486423713, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94f3ccf9f8a638fa974e1f69627832ef, server=bd6592c37bb2,40087,1732486423713 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T22:15:36,899 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732486521610.94f3ccf9f8a638fa974e1f69627832ef., hostname=bd6592c37bb2,40087,1732486423713, seqNum=5 because the exception is null or not the one we care about 2024-11-24T22:15:36,899 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=35.78 KB at sequenceid=175 (bloomFilter=true), to=hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/94f3ccf9f8a638fa974e1f69627832ef/.tmp/C/c68109f6bef04e81a21e61cdb622d3f0 2024-11-24T22:15:36,903 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/94f3ccf9f8a638fa974e1f69627832ef/.tmp/A/e25b282932044a15acb1a40c17957984 as hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/94f3ccf9f8a638fa974e1f69627832ef/A/e25b282932044a15acb1a40c17957984 2024-11-24T22:15:36,906 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/94f3ccf9f8a638fa974e1f69627832ef/A/e25b282932044a15acb1a40c17957984, entries=200, sequenceid=175, filesize=38.8 K 2024-11-24T22:15:36,907 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/94f3ccf9f8a638fa974e1f69627832ef/.tmp/B/853e51745e104112b072f41f1a4b6852 as hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/94f3ccf9f8a638fa974e1f69627832ef/B/853e51745e104112b072f41f1a4b6852 2024-11-24T22:15:36,910 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/94f3ccf9f8a638fa974e1f69627832ef/B/853e51745e104112b072f41f1a4b6852, entries=150, sequenceid=175, filesize=11.9 K 2024-11-24T22:15:36,911 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/94f3ccf9f8a638fa974e1f69627832ef/.tmp/C/c68109f6bef04e81a21e61cdb622d3f0 as hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/94f3ccf9f8a638fa974e1f69627832ef/C/c68109f6bef04e81a21e61cdb622d3f0 2024-11-24T22:15:36,913 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40087 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94f3ccf9f8a638fa974e1f69627832ef, server=bd6592c37bb2,40087,1732486423713 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-24T22:15:36,914 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/94f3ccf9f8a638fa974e1f69627832ef/C/c68109f6bef04e81a21e61cdb622d3f0, entries=150, sequenceid=175, filesize=11.9 K 2024-11-24T22:15:36,914 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40087 {}] ipc.CallRunner(138): callId: 60 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:32810 deadline: 1732486596901, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94f3ccf9f8a638fa974e1f69627832ef, server=bd6592c37bb2,40087,1732486423713 2024-11-24T22:15:36,914 WARN [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRpcRetryingCaller(177): Call to bd6592c37bb2,40087,1732486423713 for 'test_row_2' in 94f3ccf9f8a638fa974e1f69627832ef of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2192 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94f3ccf9f8a638fa974e1f69627832ef, server=bd6592c37bb2,40087,1732486423713 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor44.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94f3ccf9f8a638fa974e1f69627832ef, server=bd6592c37bb2,40087,1732486423713 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-24T22:15:36,914 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732486521610.94f3ccf9f8a638fa974e1f69627832ef., hostname=bd6592c37bb2,40087,1732486423713, seqNum=5 , the old value is region=TestAcidGuarantees,,1732486521610.94f3ccf9f8a638fa974e1f69627832ef., hostname=bd6592c37bb2,40087,1732486423713, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94f3ccf9f8a638fa974e1f69627832ef, server=bd6592c37bb2,40087,1732486423713 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T22:15:36,915 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732486521610.94f3ccf9f8a638fa974e1f69627832ef., hostname=bd6592c37bb2,40087,1732486423713, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94f3ccf9f8a638fa974e1f69627832ef, server=bd6592c37bb2,40087,1732486423713 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T22:15:36,915 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732486521610.94f3ccf9f8a638fa974e1f69627832ef., hostname=bd6592c37bb2,40087,1732486423713, seqNum=5 because the exception is null or not the one we care about 2024-11-24T22:15:36,915 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~107.34 KB/109920, heapSize ~319.45 KB/327120, currentSize=67.09 KB/68700 for 94f3ccf9f8a638fa974e1f69627832ef in 1258ms, sequenceid=175, compaction requested=true 2024-11-24T22:15:36,915 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 94f3ccf9f8a638fa974e1f69627832ef: 2024-11-24T22:15:36,915 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 94f3ccf9f8a638fa974e1f69627832ef:A, priority=-2147483648, current under compaction store size is 1 2024-11-24T22:15:36,915 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-24T22:15:36,915 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 94f3ccf9f8a638fa974e1f69627832ef:B, priority=-2147483648, current under compaction store size is 2 2024-11-24T22:15:36,915 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-24T22:15:36,915 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 94f3ccf9f8a638fa974e1f69627832ef:C, priority=-2147483648, current under compaction store size is 3 2024-11-24T22:15:36,915 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-24T22:15:36,915 DEBUG [RS:0;bd6592c37bb2:40087-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-24T22:15:36,915 DEBUG [RS:0;bd6592c37bb2:40087-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-24T22:15:36,915 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40087 {}] regionserver.HRegion(8855): Flush requested on 94f3ccf9f8a638fa974e1f69627832ef 2024-11-24T22:15:36,915 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 94f3ccf9f8a638fa974e1f69627832ef 3/3 column families, dataSize=73.80 KB heapSize=219.89 KB 2024-11-24T22:15:36,916 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 94f3ccf9f8a638fa974e1f69627832ef, store=A 2024-11-24T22:15:36,916 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-24T22:15:36,916 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 94f3ccf9f8a638fa974e1f69627832ef, store=B 2024-11-24T22:15:36,916 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-24T22:15:36,916 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 94f3ccf9f8a638fa974e1f69627832ef, store=C 2024-11-24T22:15:36,916 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-24T22:15:36,918 DEBUG [RS:0;bd6592c37bb2:40087-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 48762 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-24T22:15:36,918 DEBUG [RS:0;bd6592c37bb2:40087-longCompactions-0 {}] regionserver.HStore(1541): 94f3ccf9f8a638fa974e1f69627832ef/B is initiating minor compaction (all files) 2024-11-24T22:15:36,918 INFO [RS:0;bd6592c37bb2:40087-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 94f3ccf9f8a638fa974e1f69627832ef/B in TestAcidGuarantees,,1732486521610.94f3ccf9f8a638fa974e1f69627832ef. 2024-11-24T22:15:36,918 DEBUG [RS:0;bd6592c37bb2:40087-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 141866 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-24T22:15:36,918 DEBUG [RS:0;bd6592c37bb2:40087-shortCompactions-0 {}] regionserver.HStore(1541): 94f3ccf9f8a638fa974e1f69627832ef/A is initiating minor compaction (all files) 2024-11-24T22:15:36,918 INFO [RS:0;bd6592c37bb2:40087-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/94f3ccf9f8a638fa974e1f69627832ef/B/77b19474d2d548f7916c60b8c12adad0, hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/94f3ccf9f8a638fa974e1f69627832ef/B/d141ed51dbd0448f9084a2a5552ef064, hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/94f3ccf9f8a638fa974e1f69627832ef/B/aed5d43a422c4c0a892806e5dc6d9ddb, hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/94f3ccf9f8a638fa974e1f69627832ef/B/853e51745e104112b072f41f1a4b6852] into tmpdir=hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/94f3ccf9f8a638fa974e1f69627832ef/.tmp, totalSize=47.6 K 2024-11-24T22:15:36,918 INFO [RS:0;bd6592c37bb2:40087-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 94f3ccf9f8a638fa974e1f69627832ef/A in TestAcidGuarantees,,1732486521610.94f3ccf9f8a638fa974e1f69627832ef. 2024-11-24T22:15:36,919 INFO [RS:0;bd6592c37bb2:40087-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/94f3ccf9f8a638fa974e1f69627832ef/A/fce104982b484448966f2a223c8fc392, hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/94f3ccf9f8a638fa974e1f69627832ef/A/697a1699f64d45fcb65780147612779b, hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/94f3ccf9f8a638fa974e1f69627832ef/A/c3b3073483d94256bdd7bddae194779a, hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/94f3ccf9f8a638fa974e1f69627832ef/A/e25b282932044a15acb1a40c17957984] into tmpdir=hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/94f3ccf9f8a638fa974e1f69627832ef/.tmp, totalSize=138.5 K 2024-11-24T22:15:36,919 INFO [RS:0;bd6592c37bb2:40087-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=12 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1732486521610.94f3ccf9f8a638fa974e1f69627832ef. 2024-11-24T22:15:36,919 DEBUG [RS:0;bd6592c37bb2:40087-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1732486521610.94f3ccf9f8a638fa974e1f69627832ef. files: [hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/94f3ccf9f8a638fa974e1f69627832ef/A/fce104982b484448966f2a223c8fc392, hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/94f3ccf9f8a638fa974e1f69627832ef/A/697a1699f64d45fcb65780147612779b, hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/94f3ccf9f8a638fa974e1f69627832ef/A/c3b3073483d94256bdd7bddae194779a, hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/94f3ccf9f8a638fa974e1f69627832ef/A/e25b282932044a15acb1a40c17957984] 2024-11-24T22:15:36,921 DEBUG [RS:0;bd6592c37bb2:40087-longCompactions-0 {}] compactions.Compactor(225): Compacting 77b19474d2d548f7916c60b8c12adad0, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=120, earliestPutTs=1732486530255 2024-11-24T22:15:36,921 DEBUG [RS:0;bd6592c37bb2:40087-shortCompactions-0 {}] compactions.Compactor(225): Compacting fce104982b484448966f2a223c8fc392, keycount=150, bloomtype=ROW, size=30.5 K, encoding=NONE, compression=NONE, seqNum=120, earliestPutTs=1732486530255 2024-11-24T22:15:36,921 DEBUG [RS:0;bd6592c37bb2:40087-longCompactions-0 {}] compactions.Compactor(225): Compacting d141ed51dbd0448f9084a2a5552ef064, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=141, earliestPutTs=1732486531374 2024-11-24T22:15:36,921 DEBUG [RS:0;bd6592c37bb2:40087-shortCompactions-0 {}] compactions.Compactor(225): Compacting 697a1699f64d45fcb65780147612779b, keycount=200, bloomtype=ROW, size=38.8 K, encoding=NONE, compression=NONE, seqNum=141, earliestPutTs=1732486531374 2024-11-24T22:15:36,921 DEBUG [RS:0;bd6592c37bb2:40087-longCompactions-0 {}] compactions.Compactor(225): Compacting aed5d43a422c4c0a892806e5dc6d9ddb, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=156, earliestPutTs=1732486532456 2024-11-24T22:15:36,921 DEBUG [RS:0;bd6592c37bb2:40087-shortCompactions-0 {}] compactions.Compactor(225): Compacting c3b3073483d94256bdd7bddae194779a, keycount=150, bloomtype=ROW, size=30.4 K, encoding=NONE, compression=NONE, seqNum=156, earliestPutTs=1732486532456 2024-11-24T22:15:36,922 DEBUG [RS:0;bd6592c37bb2:40087-longCompactions-0 {}] compactions.Compactor(225): Compacting 853e51745e104112b072f41f1a4b6852, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=175, earliestPutTs=1732486534720 2024-11-24T22:15:36,922 DEBUG [RS:0;bd6592c37bb2:40087-shortCompactions-0 {}] compactions.Compactor(225): Compacting e25b282932044a15acb1a40c17957984, keycount=200, bloomtype=ROW, size=38.8 K, encoding=NONE, compression=NONE, seqNum=175, earliestPutTs=1732486534720 2024-11-24T22:15:36,928 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411249b392706f214429eaa46998773c08a09_94f3ccf9f8a638fa974e1f69627832ef is 50, key is test_row_0/A:col10/1732486536914/Put/seqid=0 2024-11-24T22:15:36,931 INFO [RS:0;bd6592c37bb2:40087-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=94f3ccf9f8a638fa974e1f69627832ef] 2024-11-24T22:15:36,933 INFO [RS:0;bd6592c37bb2:40087-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 94f3ccf9f8a638fa974e1f69627832ef#B#compaction#339 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-24T22:15:36,933 DEBUG [RS:0;bd6592c37bb2:40087-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/94f3ccf9f8a638fa974e1f69627832ef/.tmp/B/bb9965a6ef4f4109a98514be227d61cf is 50, key is test_row_0/B:col10/1732486534720/Put/seqid=0 2024-11-24T22:15:36,936 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45929 is added to blk_1073742228_1404 (size=14794) 2024-11-24T22:15:36,947 DEBUG [RS:0;bd6592c37bb2:40087-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e202411245716991265f945e28842a6540ea15b6d_94f3ccf9f8a638fa974e1f69627832ef store=[table=TestAcidGuarantees family=A region=94f3ccf9f8a638fa974e1f69627832ef] 2024-11-24T22:15:36,948 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45929 is added to blk_1073742229_1405 (size=12595) 2024-11-24T22:15:36,949 DEBUG [RS:0;bd6592c37bb2:40087-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e202411245716991265f945e28842a6540ea15b6d_94f3ccf9f8a638fa974e1f69627832ef, store=[table=TestAcidGuarantees family=A region=94f3ccf9f8a638fa974e1f69627832ef] 2024-11-24T22:15:36,950 DEBUG [RS:0;bd6592c37bb2:40087-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411245716991265f945e28842a6540ea15b6d_94f3ccf9f8a638fa974e1f69627832ef because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=94f3ccf9f8a638fa974e1f69627832ef] 2024-11-24T22:15:36,962 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45929 is added to blk_1073742230_1406 (size=4469) 2024-11-24T22:15:37,064 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40087 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94f3ccf9f8a638fa974e1f69627832ef, server=bd6592c37bb2,40087,1732486423713 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-24T22:15:37,064 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40087 {}] ipc.CallRunner(138): callId: 65 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:32846 deadline: 1732486597048, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94f3ccf9f8a638fa974e1f69627832ef, server=bd6592c37bb2,40087,1732486423713 2024-11-24T22:15:37,064 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40087 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94f3ccf9f8a638fa974e1f69627832ef, server=bd6592c37bb2,40087,1732486423713 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-24T22:15:37,064 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40087 {}] ipc.CallRunner(138): callId: 65 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:32774 deadline: 1732486597050, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94f3ccf9f8a638fa974e1f69627832ef, server=bd6592c37bb2,40087,1732486423713 2024-11-24T22:15:37,064 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732486521610.94f3ccf9f8a638fa974e1f69627832ef., hostname=bd6592c37bb2,40087,1732486423713, seqNum=5 , the old value is region=TestAcidGuarantees,,1732486521610.94f3ccf9f8a638fa974e1f69627832ef., hostname=bd6592c37bb2,40087,1732486423713, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94f3ccf9f8a638fa974e1f69627832ef, server=bd6592c37bb2,40087,1732486423713 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T22:15:37,064 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732486521610.94f3ccf9f8a638fa974e1f69627832ef., hostname=bd6592c37bb2,40087,1732486423713, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94f3ccf9f8a638fa974e1f69627832ef, server=bd6592c37bb2,40087,1732486423713 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T22:15:37,064 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732486521610.94f3ccf9f8a638fa974e1f69627832ef., hostname=bd6592c37bb2,40087,1732486423713, seqNum=5 because the exception is null or not the one we care about 2024-11-24T22:15:37,065 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732486521610.94f3ccf9f8a638fa974e1f69627832ef., hostname=bd6592c37bb2,40087,1732486423713, seqNum=5 , the old value is region=TestAcidGuarantees,,1732486521610.94f3ccf9f8a638fa974e1f69627832ef., hostname=bd6592c37bb2,40087,1732486423713, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94f3ccf9f8a638fa974e1f69627832ef, server=bd6592c37bb2,40087,1732486423713 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T22:15:37,065 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732486521610.94f3ccf9f8a638fa974e1f69627832ef., hostname=bd6592c37bb2,40087,1732486423713, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94f3ccf9f8a638fa974e1f69627832ef, server=bd6592c37bb2,40087,1732486423713 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T22:15:37,065 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732486521610.94f3ccf9f8a638fa974e1f69627832ef., hostname=bd6592c37bb2,40087,1732486423713, seqNum=5 because the exception is null or not the one we care about 2024-11-24T22:15:37,066 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40087 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94f3ccf9f8a638fa974e1f69627832ef, server=bd6592c37bb2,40087,1732486423713 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-24T22:15:37,066 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40087 {}] ipc.CallRunner(138): callId: 49 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:32832 deadline: 1732486597051, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94f3ccf9f8a638fa974e1f69627832ef, server=bd6592c37bb2,40087,1732486423713 2024-11-24T22:15:37,066 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732486521610.94f3ccf9f8a638fa974e1f69627832ef., hostname=bd6592c37bb2,40087,1732486423713, seqNum=5 , the old value is region=TestAcidGuarantees,,1732486521610.94f3ccf9f8a638fa974e1f69627832ef., hostname=bd6592c37bb2,40087,1732486423713, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94f3ccf9f8a638fa974e1f69627832ef, server=bd6592c37bb2,40087,1732486423713 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T22:15:37,066 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732486521610.94f3ccf9f8a638fa974e1f69627832ef., hostname=bd6592c37bb2,40087,1732486423713, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94f3ccf9f8a638fa974e1f69627832ef, server=bd6592c37bb2,40087,1732486423713 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T22:15:37,066 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732486521610.94f3ccf9f8a638fa974e1f69627832ef., hostname=bd6592c37bb2,40087,1732486423713, seqNum=5 because the exception is null or not the one we care about 2024-11-24T22:15:37,173 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40087 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94f3ccf9f8a638fa974e1f69627832ef, server=bd6592c37bb2,40087,1732486423713 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-24T22:15:37,173 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40087 {}] ipc.CallRunner(138): callId: 66 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:32846 deadline: 1732486597171, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94f3ccf9f8a638fa974e1f69627832ef, server=bd6592c37bb2,40087,1732486423713 2024-11-24T22:15:37,173 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40087 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94f3ccf9f8a638fa974e1f69627832ef, server=bd6592c37bb2,40087,1732486423713 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-24T22:15:37,173 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40087 {}] ipc.CallRunner(138): callId: 66 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:32774 deadline: 1732486597171, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94f3ccf9f8a638fa974e1f69627832ef, server=bd6592c37bb2,40087,1732486423713 2024-11-24T22:15:37,173 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732486521610.94f3ccf9f8a638fa974e1f69627832ef., hostname=bd6592c37bb2,40087,1732486423713, seqNum=5 , the old value is region=TestAcidGuarantees,,1732486521610.94f3ccf9f8a638fa974e1f69627832ef., hostname=bd6592c37bb2,40087,1732486423713, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94f3ccf9f8a638fa974e1f69627832ef, server=bd6592c37bb2,40087,1732486423713 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T22:15:37,173 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732486521610.94f3ccf9f8a638fa974e1f69627832ef., hostname=bd6592c37bb2,40087,1732486423713, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94f3ccf9f8a638fa974e1f69627832ef, server=bd6592c37bb2,40087,1732486423713 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T22:15:37,173 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732486521610.94f3ccf9f8a638fa974e1f69627832ef., hostname=bd6592c37bb2,40087,1732486423713, seqNum=5 because the exception is null or not the one we care about 2024-11-24T22:15:37,173 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732486521610.94f3ccf9f8a638fa974e1f69627832ef., hostname=bd6592c37bb2,40087,1732486423713, seqNum=5 , the old value is region=TestAcidGuarantees,,1732486521610.94f3ccf9f8a638fa974e1f69627832ef., hostname=bd6592c37bb2,40087,1732486423713, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94f3ccf9f8a638fa974e1f69627832ef, server=bd6592c37bb2,40087,1732486423713 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T22:15:37,173 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40087 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94f3ccf9f8a638fa974e1f69627832ef, server=bd6592c37bb2,40087,1732486423713 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-24T22:15:37,174 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732486521610.94f3ccf9f8a638fa974e1f69627832ef., hostname=bd6592c37bb2,40087,1732486423713, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94f3ccf9f8a638fa974e1f69627832ef, server=bd6592c37bb2,40087,1732486423713 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T22:15:37,174 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732486521610.94f3ccf9f8a638fa974e1f69627832ef., hostname=bd6592c37bb2,40087,1732486423713, seqNum=5 because the exception is null or not the one we care about 2024-11-24T22:15:37,174 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40087 {}] ipc.CallRunner(138): callId: 50 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:32832 deadline: 1732486597171, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94f3ccf9f8a638fa974e1f69627832ef, server=bd6592c37bb2,40087,1732486423713 2024-11-24T22:15:37,174 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732486521610.94f3ccf9f8a638fa974e1f69627832ef., hostname=bd6592c37bb2,40087,1732486423713, seqNum=5 , the old value is region=TestAcidGuarantees,,1732486521610.94f3ccf9f8a638fa974e1f69627832ef., hostname=bd6592c37bb2,40087,1732486423713, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94f3ccf9f8a638fa974e1f69627832ef, server=bd6592c37bb2,40087,1732486423713 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T22:15:37,174 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732486521610.94f3ccf9f8a638fa974e1f69627832ef., hostname=bd6592c37bb2,40087,1732486423713, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94f3ccf9f8a638fa974e1f69627832ef, server=bd6592c37bb2,40087,1732486423713 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T22:15:37,174 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732486521610.94f3ccf9f8a638fa974e1f69627832ef., hostname=bd6592c37bb2,40087,1732486423713, seqNum=5 because the exception is null or not the one we care about 2024-11-24T22:15:37,201 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37701 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=101 2024-11-24T22:15:37,201 INFO [RPCClient-NioEventLoopGroup-4-8 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-24T22:15:37,202 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37701 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-24T22:15:37,203 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37701 {}] procedure2.ProcedureExecutor(1139): Stored pid=103, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=103, table=TestAcidGuarantees 2024-11-24T22:15:37,203 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37701 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=103 2024-11-24T22:15:37,204 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=103, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=103, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-24T22:15:37,204 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=103, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=103, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-24T22:15:37,204 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=104, ppid=103, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-24T22:15:37,311 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37701 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=103 2024-11-24T22:15:37,337 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T22:15:37,340 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411249b392706f214429eaa46998773c08a09_94f3ccf9f8a638fa974e1f69627832ef to hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411249b392706f214429eaa46998773c08a09_94f3ccf9f8a638fa974e1f69627832ef 2024-11-24T22:15:37,341 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/94f3ccf9f8a638fa974e1f69627832ef/.tmp/A/28d65c9086ac4081b22ae0ce11933e15, store: [table=TestAcidGuarantees family=A region=94f3ccf9f8a638fa974e1f69627832ef] 2024-11-24T22:15:37,342 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/94f3ccf9f8a638fa974e1f69627832ef/.tmp/A/28d65c9086ac4081b22ae0ce11933e15 is 175, key is test_row_0/A:col10/1732486536914/Put/seqid=0 2024-11-24T22:15:37,345 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45929 is added to blk_1073742231_1407 (size=39749) 2024-11-24T22:15:37,352 DEBUG [RS:0;bd6592c37bb2:40087-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/94f3ccf9f8a638fa974e1f69627832ef/.tmp/B/bb9965a6ef4f4109a98514be227d61cf as hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/94f3ccf9f8a638fa974e1f69627832ef/B/bb9965a6ef4f4109a98514be227d61cf 2024-11-24T22:15:37,356 INFO [RS:0;bd6592c37bb2:40087-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 94f3ccf9f8a638fa974e1f69627832ef/B of 94f3ccf9f8a638fa974e1f69627832ef into bb9965a6ef4f4109a98514be227d61cf(size=12.3 K), total size for store is 12.3 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-24T22:15:37,356 DEBUG [RS:0;bd6592c37bb2:40087-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 94f3ccf9f8a638fa974e1f69627832ef: 2024-11-24T22:15:37,356 INFO [RS:0;bd6592c37bb2:40087-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732486521610.94f3ccf9f8a638fa974e1f69627832ef., storeName=94f3ccf9f8a638fa974e1f69627832ef/B, priority=12, startTime=1732486536915; duration=0sec 2024-11-24T22:15:37,356 DEBUG [RS:0;bd6592c37bb2:40087-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-24T22:15:37,356 DEBUG [RS:0;bd6592c37bb2:40087-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 94f3ccf9f8a638fa974e1f69627832ef:B 2024-11-24T22:15:37,356 DEBUG [RS:0;bd6592c37bb2:40087-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-24T22:15:37,357 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=40087 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=104 2024-11-24T22:15:37,357 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bd6592c37bb2:0-1 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732486521610.94f3ccf9f8a638fa974e1f69627832ef. 2024-11-24T22:15:37,357 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bd6592c37bb2:0-1 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732486521610.94f3ccf9f8a638fa974e1f69627832ef. as already flushing 2024-11-24T22:15:37,357 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bd6592c37bb2:0-1 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732486521610.94f3ccf9f8a638fa974e1f69627832ef. 2024-11-24T22:15:37,357 ERROR [RS_FLUSH_OPERATIONS-regionserver/bd6592c37bb2:0-1 {event_type=RS_FLUSH_REGIONS, pid=104}] handler.RSProcedureHandler(58): pid=104 java.io.IOException: Unable to complete flush {ENCODED => 94f3ccf9f8a638fa974e1f69627832ef, NAME => 'TestAcidGuarantees,,1732486521610.94f3ccf9f8a638fa974e1f69627832ef.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T22:15:37,357 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bd6592c37bb2:0-1 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=104 java.io.IOException: Unable to complete flush {ENCODED => 94f3ccf9f8a638fa974e1f69627832ef, NAME => 'TestAcidGuarantees,,1732486521610.94f3ccf9f8a638fa974e1f69627832ef.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T22:15:37,358 DEBUG [RS:0;bd6592c37bb2:40087-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 48728 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-24T22:15:37,358 DEBUG [RS:0;bd6592c37bb2:40087-longCompactions-0 {}] regionserver.HStore(1541): 94f3ccf9f8a638fa974e1f69627832ef/C is initiating minor compaction (all files) 2024-11-24T22:15:37,358 INFO [RS:0;bd6592c37bb2:40087-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 94f3ccf9f8a638fa974e1f69627832ef/C in TestAcidGuarantees,,1732486521610.94f3ccf9f8a638fa974e1f69627832ef. 2024-11-24T22:15:37,358 INFO [RS:0;bd6592c37bb2:40087-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/94f3ccf9f8a638fa974e1f69627832ef/C/cb3ac673639340e892a044d258cbd444, hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/94f3ccf9f8a638fa974e1f69627832ef/C/a3395d0e4c1c43a98e2163a3d14415df, hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/94f3ccf9f8a638fa974e1f69627832ef/C/a12ca2c795164aae8da91298a1edd817, hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/94f3ccf9f8a638fa974e1f69627832ef/C/c68109f6bef04e81a21e61cdb622d3f0] into tmpdir=hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/94f3ccf9f8a638fa974e1f69627832ef/.tmp, totalSize=47.6 K 2024-11-24T22:15:37,358 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37701 {}] master.HMaster(4177): Remote procedure failed, pid=104 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 94f3ccf9f8a638fa974e1f69627832ef, NAME => 'TestAcidGuarantees,,1732486521610.94f3ccf9f8a638fa974e1f69627832ef.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 94f3ccf9f8a638fa974e1f69627832ef, NAME => 'TestAcidGuarantees,,1732486521610.94f3ccf9f8a638fa974e1f69627832ef.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T22:15:37,359 DEBUG [RS:0;bd6592c37bb2:40087-longCompactions-0 {}] compactions.Compactor(225): Compacting cb3ac673639340e892a044d258cbd444, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=120, earliestPutTs=1732486530255 2024-11-24T22:15:37,359 DEBUG [RS:0;bd6592c37bb2:40087-longCompactions-0 {}] compactions.Compactor(225): Compacting a3395d0e4c1c43a98e2163a3d14415df, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=141, earliestPutTs=1732486531374 2024-11-24T22:15:37,359 DEBUG [RS:0;bd6592c37bb2:40087-longCompactions-0 {}] compactions.Compactor(225): Compacting a12ca2c795164aae8da91298a1edd817, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=156, earliestPutTs=1732486532456 2024-11-24T22:15:37,360 DEBUG [RS:0;bd6592c37bb2:40087-longCompactions-0 {}] compactions.Compactor(225): Compacting c68109f6bef04e81a21e61cdb622d3f0, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=175, earliestPutTs=1732486534720 2024-11-24T22:15:37,363 INFO [RS:0;bd6592c37bb2:40087-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 94f3ccf9f8a638fa974e1f69627832ef#A#compaction#340 average throughput is 0.06 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-24T22:15:37,363 DEBUG [RS:0;bd6592c37bb2:40087-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/94f3ccf9f8a638fa974e1f69627832ef/.tmp/A/17c6ecafbb0d4ff6b176abbe1aed6b68 is 175, key is test_row_0/A:col10/1732486534720/Put/seqid=0 2024-11-24T22:15:37,370 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45929 is added to blk_1073742232_1408 (size=31549) 2024-11-24T22:15:37,371 INFO [RS:0;bd6592c37bb2:40087-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 94f3ccf9f8a638fa974e1f69627832ef#C#compaction#341 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-24T22:15:37,372 DEBUG [RS:0;bd6592c37bb2:40087-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/94f3ccf9f8a638fa974e1f69627832ef/.tmp/C/d067a887518d4c33a3df8668caf4c2da is 50, key is test_row_0/C:col10/1732486534720/Put/seqid=0 2024-11-24T22:15:37,375 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45929 is added to blk_1073742233_1409 (size=12561) 2024-11-24T22:15:37,376 DEBUG [RS:0;bd6592c37bb2:40087-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/94f3ccf9f8a638fa974e1f69627832ef/.tmp/A/17c6ecafbb0d4ff6b176abbe1aed6b68 as hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/94f3ccf9f8a638fa974e1f69627832ef/A/17c6ecafbb0d4ff6b176abbe1aed6b68 2024-11-24T22:15:37,379 DEBUG [RS:0;bd6592c37bb2:40087-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/94f3ccf9f8a638fa974e1f69627832ef/.tmp/C/d067a887518d4c33a3df8668caf4c2da as hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/94f3ccf9f8a638fa974e1f69627832ef/C/d067a887518d4c33a3df8668caf4c2da 2024-11-24T22:15:37,380 INFO [RS:0;bd6592c37bb2:40087-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 94f3ccf9f8a638fa974e1f69627832ef/A of 94f3ccf9f8a638fa974e1f69627832ef into 17c6ecafbb0d4ff6b176abbe1aed6b68(size=30.8 K), total size for store is 30.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-24T22:15:37,380 DEBUG [RS:0;bd6592c37bb2:40087-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 94f3ccf9f8a638fa974e1f69627832ef: 2024-11-24T22:15:37,380 INFO [RS:0;bd6592c37bb2:40087-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732486521610.94f3ccf9f8a638fa974e1f69627832ef., storeName=94f3ccf9f8a638fa974e1f69627832ef/A, priority=12, startTime=1732486536915; duration=0sec 2024-11-24T22:15:37,380 DEBUG [RS:0;bd6592c37bb2:40087-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-24T22:15:37,380 DEBUG [RS:0;bd6592c37bb2:40087-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 94f3ccf9f8a638fa974e1f69627832ef:A 2024-11-24T22:15:37,382 INFO [RS:0;bd6592c37bb2:40087-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 94f3ccf9f8a638fa974e1f69627832ef/C of 94f3ccf9f8a638fa974e1f69627832ef into d067a887518d4c33a3df8668caf4c2da(size=12.3 K), total size for store is 12.3 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-24T22:15:37,382 DEBUG [RS:0;bd6592c37bb2:40087-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 94f3ccf9f8a638fa974e1f69627832ef: 2024-11-24T22:15:37,382 INFO [RS:0;bd6592c37bb2:40087-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732486521610.94f3ccf9f8a638fa974e1f69627832ef., storeName=94f3ccf9f8a638fa974e1f69627832ef/C, priority=12, startTime=1732486536915; duration=0sec 2024-11-24T22:15:37,383 DEBUG [RS:0;bd6592c37bb2:40087-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-24T22:15:37,383 DEBUG [RS:0;bd6592c37bb2:40087-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 94f3ccf9f8a638fa974e1f69627832ef:C 2024-11-24T22:15:37,390 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40087 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94f3ccf9f8a638fa974e1f69627832ef, server=bd6592c37bb2,40087,1732486423713 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-24T22:15:37,391 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40087 {}] ipc.CallRunner(138): callId: 67 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:32846 deadline: 1732486597381, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94f3ccf9f8a638fa974e1f69627832ef, server=bd6592c37bb2,40087,1732486423713 2024-11-24T22:15:37,391 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40087 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94f3ccf9f8a638fa974e1f69627832ef, server=bd6592c37bb2,40087,1732486423713 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-24T22:15:37,391 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732486521610.94f3ccf9f8a638fa974e1f69627832ef., hostname=bd6592c37bb2,40087,1732486423713, seqNum=5 , the old value is region=TestAcidGuarantees,,1732486521610.94f3ccf9f8a638fa974e1f69627832ef., hostname=bd6592c37bb2,40087,1732486423713, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94f3ccf9f8a638fa974e1f69627832ef, server=bd6592c37bb2,40087,1732486423713 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T22:15:37,391 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732486521610.94f3ccf9f8a638fa974e1f69627832ef., hostname=bd6592c37bb2,40087,1732486423713, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94f3ccf9f8a638fa974e1f69627832ef, server=bd6592c37bb2,40087,1732486423713 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T22:15:37,391 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40087 {}] ipc.CallRunner(138): callId: 67 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:32774 deadline: 1732486597381, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94f3ccf9f8a638fa974e1f69627832ef, server=bd6592c37bb2,40087,1732486423713 2024-11-24T22:15:37,391 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732486521610.94f3ccf9f8a638fa974e1f69627832ef., hostname=bd6592c37bb2,40087,1732486423713, seqNum=5 because the exception is null or not the one we care about 2024-11-24T22:15:37,391 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40087 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94f3ccf9f8a638fa974e1f69627832ef, server=bd6592c37bb2,40087,1732486423713 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-24T22:15:37,391 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732486521610.94f3ccf9f8a638fa974e1f69627832ef., hostname=bd6592c37bb2,40087,1732486423713, seqNum=5 , the old value is region=TestAcidGuarantees,,1732486521610.94f3ccf9f8a638fa974e1f69627832ef., hostname=bd6592c37bb2,40087,1732486423713, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94f3ccf9f8a638fa974e1f69627832ef, server=bd6592c37bb2,40087,1732486423713 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T22:15:37,391 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732486521610.94f3ccf9f8a638fa974e1f69627832ef., hostname=bd6592c37bb2,40087,1732486423713, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94f3ccf9f8a638fa974e1f69627832ef, server=bd6592c37bb2,40087,1732486423713 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T22:15:37,391 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40087 {}] ipc.CallRunner(138): callId: 51 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:32832 deadline: 1732486597381, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94f3ccf9f8a638fa974e1f69627832ef, server=bd6592c37bb2,40087,1732486423713 2024-11-24T22:15:37,391 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732486521610.94f3ccf9f8a638fa974e1f69627832ef., hostname=bd6592c37bb2,40087,1732486423713, seqNum=5 because the exception is null or not the one we care about 2024-11-24T22:15:37,391 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732486521610.94f3ccf9f8a638fa974e1f69627832ef., hostname=bd6592c37bb2,40087,1732486423713, seqNum=5 , the old value is region=TestAcidGuarantees,,1732486521610.94f3ccf9f8a638fa974e1f69627832ef., hostname=bd6592c37bb2,40087,1732486423713, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94f3ccf9f8a638fa974e1f69627832ef, server=bd6592c37bb2,40087,1732486423713 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T22:15:37,392 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732486521610.94f3ccf9f8a638fa974e1f69627832ef., hostname=bd6592c37bb2,40087,1732486423713, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94f3ccf9f8a638fa974e1f69627832ef, server=bd6592c37bb2,40087,1732486423713 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T22:15:37,392 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732486521610.94f3ccf9f8a638fa974e1f69627832ef., hostname=bd6592c37bb2,40087,1732486423713, seqNum=5 because the exception is null or not the one we care about 2024-11-24T22:15:37,510 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=40087 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=104 2024-11-24T22:15:37,510 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bd6592c37bb2:0-2 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732486521610.94f3ccf9f8a638fa974e1f69627832ef. 2024-11-24T22:15:37,510 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bd6592c37bb2:0-2 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732486521610.94f3ccf9f8a638fa974e1f69627832ef. as already flushing 2024-11-24T22:15:37,510 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bd6592c37bb2:0-2 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732486521610.94f3ccf9f8a638fa974e1f69627832ef. 2024-11-24T22:15:37,510 ERROR [RS_FLUSH_OPERATIONS-regionserver/bd6592c37bb2:0-2 {event_type=RS_FLUSH_REGIONS, pid=104}] handler.RSProcedureHandler(58): pid=104 java.io.IOException: Unable to complete flush {ENCODED => 94f3ccf9f8a638fa974e1f69627832ef, NAME => 'TestAcidGuarantees,,1732486521610.94f3ccf9f8a638fa974e1f69627832ef.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T22:15:37,511 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bd6592c37bb2:0-2 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=104 java.io.IOException: Unable to complete flush {ENCODED => 94f3ccf9f8a638fa974e1f69627832ef, NAME => 'TestAcidGuarantees,,1732486521610.94f3ccf9f8a638fa974e1f69627832ef.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T22:15:37,511 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37701 {}] master.HMaster(4177): Remote procedure failed, pid=104 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 94f3ccf9f8a638fa974e1f69627832ef, NAME => 'TestAcidGuarantees,,1732486521610.94f3ccf9f8a638fa974e1f69627832ef.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 94f3ccf9f8a638fa974e1f69627832ef, NAME => 'TestAcidGuarantees,,1732486521610.94f3ccf9f8a638fa974e1f69627832ef.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T22:15:37,520 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37701 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=103 2024-11-24T22:15:37,664 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=40087 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=104 2024-11-24T22:15:37,664 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bd6592c37bb2:0-0 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732486521610.94f3ccf9f8a638fa974e1f69627832ef. 2024-11-24T22:15:37,664 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bd6592c37bb2:0-0 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732486521610.94f3ccf9f8a638fa974e1f69627832ef. as already flushing 2024-11-24T22:15:37,664 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bd6592c37bb2:0-0 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732486521610.94f3ccf9f8a638fa974e1f69627832ef. 2024-11-24T22:15:37,664 ERROR [RS_FLUSH_OPERATIONS-regionserver/bd6592c37bb2:0-0 {event_type=RS_FLUSH_REGIONS, pid=104}] handler.RSProcedureHandler(58): pid=104 java.io.IOException: Unable to complete flush {ENCODED => 94f3ccf9f8a638fa974e1f69627832ef, NAME => 'TestAcidGuarantees,,1732486521610.94f3ccf9f8a638fa974e1f69627832ef.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T22:15:37,664 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bd6592c37bb2:0-0 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=104 java.io.IOException: Unable to complete flush {ENCODED => 94f3ccf9f8a638fa974e1f69627832ef, NAME => 'TestAcidGuarantees,,1732486521610.94f3ccf9f8a638fa974e1f69627832ef.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T22:15:37,665 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37701 {}] master.HMaster(4177): Remote procedure failed, pid=104 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 94f3ccf9f8a638fa974e1f69627832ef, NAME => 'TestAcidGuarantees,,1732486521610.94f3ccf9f8a638fa974e1f69627832ef.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 94f3ccf9f8a638fa974e1f69627832ef, NAME => 'TestAcidGuarantees,,1732486521610.94f3ccf9f8a638fa974e1f69627832ef.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T22:15:37,712 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40087 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94f3ccf9f8a638fa974e1f69627832ef, server=bd6592c37bb2,40087,1732486423713 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-24T22:15:37,712 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40087 {}] ipc.CallRunner(138): callId: 68 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:32846 deadline: 1732486597701, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94f3ccf9f8a638fa974e1f69627832ef, server=bd6592c37bb2,40087,1732486423713 2024-11-24T22:15:37,712 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40087 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94f3ccf9f8a638fa974e1f69627832ef, server=bd6592c37bb2,40087,1732486423713 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-24T22:15:37,712 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732486521610.94f3ccf9f8a638fa974e1f69627832ef., hostname=bd6592c37bb2,40087,1732486423713, seqNum=5 , the old value is region=TestAcidGuarantees,,1732486521610.94f3ccf9f8a638fa974e1f69627832ef., hostname=bd6592c37bb2,40087,1732486423713, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94f3ccf9f8a638fa974e1f69627832ef, server=bd6592c37bb2,40087,1732486423713 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T22:15:37,712 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732486521610.94f3ccf9f8a638fa974e1f69627832ef., hostname=bd6592c37bb2,40087,1732486423713, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94f3ccf9f8a638fa974e1f69627832ef, server=bd6592c37bb2,40087,1732486423713 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T22:15:37,712 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40087 {}] ipc.CallRunner(138): callId: 68 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:32774 deadline: 1732486597701, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94f3ccf9f8a638fa974e1f69627832ef, server=bd6592c37bb2,40087,1732486423713 2024-11-24T22:15:37,712 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732486521610.94f3ccf9f8a638fa974e1f69627832ef., hostname=bd6592c37bb2,40087,1732486423713, seqNum=5 because the exception is null or not the one we care about 2024-11-24T22:15:37,713 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732486521610.94f3ccf9f8a638fa974e1f69627832ef., hostname=bd6592c37bb2,40087,1732486423713, seqNum=5 , the old value is region=TestAcidGuarantees,,1732486521610.94f3ccf9f8a638fa974e1f69627832ef., hostname=bd6592c37bb2,40087,1732486423713, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94f3ccf9f8a638fa974e1f69627832ef, server=bd6592c37bb2,40087,1732486423713 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T22:15:37,713 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732486521610.94f3ccf9f8a638fa974e1f69627832ef., hostname=bd6592c37bb2,40087,1732486423713, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94f3ccf9f8a638fa974e1f69627832ef, server=bd6592c37bb2,40087,1732486423713 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T22:15:37,713 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40087 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94f3ccf9f8a638fa974e1f69627832ef, server=bd6592c37bb2,40087,1732486423713 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-24T22:15:37,713 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732486521610.94f3ccf9f8a638fa974e1f69627832ef., hostname=bd6592c37bb2,40087,1732486423713, seqNum=5 because the exception is null or not the one we care about 2024-11-24T22:15:37,713 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40087 {}] ipc.CallRunner(138): callId: 52 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:32832 deadline: 1732486597701, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94f3ccf9f8a638fa974e1f69627832ef, server=bd6592c37bb2,40087,1732486423713 2024-11-24T22:15:37,713 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732486521610.94f3ccf9f8a638fa974e1f69627832ef., hostname=bd6592c37bb2,40087,1732486423713, seqNum=5 , the old value is region=TestAcidGuarantees,,1732486521610.94f3ccf9f8a638fa974e1f69627832ef., hostname=bd6592c37bb2,40087,1732486423713, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94f3ccf9f8a638fa974e1f69627832ef, server=bd6592c37bb2,40087,1732486423713 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T22:15:37,713 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732486521610.94f3ccf9f8a638fa974e1f69627832ef., hostname=bd6592c37bb2,40087,1732486423713, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94f3ccf9f8a638fa974e1f69627832ef, server=bd6592c37bb2,40087,1732486423713 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T22:15:37,713 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732486521610.94f3ccf9f8a638fa974e1f69627832ef., hostname=bd6592c37bb2,40087,1732486423713, seqNum=5 because the exception is null or not the one we care about 2024-11-24T22:15:37,746 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=189, memsize=24.6 K, hasBloomFilter=true, into tmp file hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/94f3ccf9f8a638fa974e1f69627832ef/.tmp/A/28d65c9086ac4081b22ae0ce11933e15 2024-11-24T22:15:37,754 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/94f3ccf9f8a638fa974e1f69627832ef/.tmp/B/72127362fbeb4a0598f8132387bbe316 is 50, key is test_row_0/B:col10/1732486536914/Put/seqid=0 2024-11-24T22:15:37,759 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45929 is added to blk_1073742234_1410 (size=12151) 2024-11-24T22:15:37,766 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=24.60 KB at sequenceid=189 (bloomFilter=true), to=hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/94f3ccf9f8a638fa974e1f69627832ef/.tmp/B/72127362fbeb4a0598f8132387bbe316 2024-11-24T22:15:37,776 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/94f3ccf9f8a638fa974e1f69627832ef/.tmp/C/644e239d0c2a44909d1456f6cd2cb2b2 is 50, key is test_row_0/C:col10/1732486536914/Put/seqid=0 2024-11-24T22:15:37,783 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45929 is added to blk_1073742235_1411 (size=12151) 2024-11-24T22:15:37,817 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=40087 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=104 2024-11-24T22:15:37,817 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bd6592c37bb2:0-1 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732486521610.94f3ccf9f8a638fa974e1f69627832ef. 2024-11-24T22:15:37,817 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bd6592c37bb2:0-1 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732486521610.94f3ccf9f8a638fa974e1f69627832ef. as already flushing 2024-11-24T22:15:37,817 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bd6592c37bb2:0-1 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732486521610.94f3ccf9f8a638fa974e1f69627832ef. 2024-11-24T22:15:37,817 ERROR [RS_FLUSH_OPERATIONS-regionserver/bd6592c37bb2:0-1 {event_type=RS_FLUSH_REGIONS, pid=104}] handler.RSProcedureHandler(58): pid=104 java.io.IOException: Unable to complete flush {ENCODED => 94f3ccf9f8a638fa974e1f69627832ef, NAME => 'TestAcidGuarantees,,1732486521610.94f3ccf9f8a638fa974e1f69627832ef.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T22:15:37,817 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bd6592c37bb2:0-1 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=104 java.io.IOException: Unable to complete flush {ENCODED => 94f3ccf9f8a638fa974e1f69627832ef, NAME => 'TestAcidGuarantees,,1732486521610.94f3ccf9f8a638fa974e1f69627832ef.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T22:15:37,818 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37701 {}] master.HMaster(4177): Remote procedure failed, pid=104 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 94f3ccf9f8a638fa974e1f69627832ef, NAME => 'TestAcidGuarantees,,1732486521610.94f3ccf9f8a638fa974e1f69627832ef.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 94f3ccf9f8a638fa974e1f69627832ef, NAME => 'TestAcidGuarantees,,1732486521610.94f3ccf9f8a638fa974e1f69627832ef.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T22:15:37,831 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37701 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=103 2024-11-24T22:15:37,921 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40087 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94f3ccf9f8a638fa974e1f69627832ef, server=bd6592c37bb2,40087,1732486423713 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-24T22:15:37,922 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40087 {}] ipc.CallRunner(138): callId: 75 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:32822 deadline: 1732486597910, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94f3ccf9f8a638fa974e1f69627832ef, server=bd6592c37bb2,40087,1732486423713 2024-11-24T22:15:37,922 WARN [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRpcRetryingCaller(177): Call to bd6592c37bb2,40087,1732486423713 for 'test_row_2' in 94f3ccf9f8a638fa974e1f69627832ef of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2181 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94f3ccf9f8a638fa974e1f69627832ef, server=bd6592c37bb2,40087,1732486423713 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor44.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94f3ccf9f8a638fa974e1f69627832ef, server=bd6592c37bb2,40087,1732486423713 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-24T22:15:37,923 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732486521610.94f3ccf9f8a638fa974e1f69627832ef., hostname=bd6592c37bb2,40087,1732486423713, seqNum=5 , the old value is region=TestAcidGuarantees,,1732486521610.94f3ccf9f8a638fa974e1f69627832ef., hostname=bd6592c37bb2,40087,1732486423713, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94f3ccf9f8a638fa974e1f69627832ef, server=bd6592c37bb2,40087,1732486423713 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T22:15:37,923 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732486521610.94f3ccf9f8a638fa974e1f69627832ef., hostname=bd6592c37bb2,40087,1732486423713, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94f3ccf9f8a638fa974e1f69627832ef, server=bd6592c37bb2,40087,1732486423713 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T22:15:37,923 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732486521610.94f3ccf9f8a638fa974e1f69627832ef., hostname=bd6592c37bb2,40087,1732486423713, seqNum=5 because the exception is null or not the one we care about 2024-11-24T22:15:37,970 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=40087 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=104 2024-11-24T22:15:37,970 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bd6592c37bb2:0-2 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732486521610.94f3ccf9f8a638fa974e1f69627832ef. 2024-11-24T22:15:37,970 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bd6592c37bb2:0-2 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732486521610.94f3ccf9f8a638fa974e1f69627832ef. as already flushing 2024-11-24T22:15:37,970 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bd6592c37bb2:0-2 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732486521610.94f3ccf9f8a638fa974e1f69627832ef. 2024-11-24T22:15:37,970 ERROR [RS_FLUSH_OPERATIONS-regionserver/bd6592c37bb2:0-2 {event_type=RS_FLUSH_REGIONS, pid=104}] handler.RSProcedureHandler(58): pid=104 java.io.IOException: Unable to complete flush {ENCODED => 94f3ccf9f8a638fa974e1f69627832ef, NAME => 'TestAcidGuarantees,,1732486521610.94f3ccf9f8a638fa974e1f69627832ef.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T22:15:37,970 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bd6592c37bb2:0-2 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=104 java.io.IOException: Unable to complete flush {ENCODED => 94f3ccf9f8a638fa974e1f69627832ef, NAME => 'TestAcidGuarantees,,1732486521610.94f3ccf9f8a638fa974e1f69627832ef.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T22:15:37,971 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37701 {}] master.HMaster(4177): Remote procedure failed, pid=104 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 94f3ccf9f8a638fa974e1f69627832ef, NAME => 'TestAcidGuarantees,,1732486521610.94f3ccf9f8a638fa974e1f69627832ef.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 94f3ccf9f8a638fa974e1f69627832ef, NAME => 'TestAcidGuarantees,,1732486521610.94f3ccf9f8a638fa974e1f69627832ef.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T22:15:38,124 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=40087 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=104 2024-11-24T22:15:38,124 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bd6592c37bb2:0-0 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732486521610.94f3ccf9f8a638fa974e1f69627832ef. 2024-11-24T22:15:38,124 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bd6592c37bb2:0-0 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732486521610.94f3ccf9f8a638fa974e1f69627832ef. as already flushing 2024-11-24T22:15:38,124 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bd6592c37bb2:0-0 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732486521610.94f3ccf9f8a638fa974e1f69627832ef. 2024-11-24T22:15:38,124 ERROR [RS_FLUSH_OPERATIONS-regionserver/bd6592c37bb2:0-0 {event_type=RS_FLUSH_REGIONS, pid=104}] handler.RSProcedureHandler(58): pid=104 java.io.IOException: Unable to complete flush {ENCODED => 94f3ccf9f8a638fa974e1f69627832ef, NAME => 'TestAcidGuarantees,,1732486521610.94f3ccf9f8a638fa974e1f69627832ef.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T22:15:38,124 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bd6592c37bb2:0-0 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=104 java.io.IOException: Unable to complete flush {ENCODED => 94f3ccf9f8a638fa974e1f69627832ef, NAME => 'TestAcidGuarantees,,1732486521610.94f3ccf9f8a638fa974e1f69627832ef.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T22:15:38,125 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37701 {}] master.HMaster(4177): Remote procedure failed, pid=104 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 94f3ccf9f8a638fa974e1f69627832ef, NAME => 'TestAcidGuarantees,,1732486521610.94f3ccf9f8a638fa974e1f69627832ef.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 94f3ccf9f8a638fa974e1f69627832ef, NAME => 'TestAcidGuarantees,,1732486521610.94f3ccf9f8a638fa974e1f69627832ef.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T22:15:38,184 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=24.60 KB at sequenceid=189 (bloomFilter=true), to=hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/94f3ccf9f8a638fa974e1f69627832ef/.tmp/C/644e239d0c2a44909d1456f6cd2cb2b2 2024-11-24T22:15:38,188 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/94f3ccf9f8a638fa974e1f69627832ef/.tmp/A/28d65c9086ac4081b22ae0ce11933e15 as hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/94f3ccf9f8a638fa974e1f69627832ef/A/28d65c9086ac4081b22ae0ce11933e15 2024-11-24T22:15:38,191 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/94f3ccf9f8a638fa974e1f69627832ef/A/28d65c9086ac4081b22ae0ce11933e15, entries=200, sequenceid=189, filesize=38.8 K 2024-11-24T22:15:38,192 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/94f3ccf9f8a638fa974e1f69627832ef/.tmp/B/72127362fbeb4a0598f8132387bbe316 as hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/94f3ccf9f8a638fa974e1f69627832ef/B/72127362fbeb4a0598f8132387bbe316 2024-11-24T22:15:38,195 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/94f3ccf9f8a638fa974e1f69627832ef/B/72127362fbeb4a0598f8132387bbe316, entries=150, sequenceid=189, filesize=11.9 K 2024-11-24T22:15:38,196 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/94f3ccf9f8a638fa974e1f69627832ef/.tmp/C/644e239d0c2a44909d1456f6cd2cb2b2 as hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/94f3ccf9f8a638fa974e1f69627832ef/C/644e239d0c2a44909d1456f6cd2cb2b2 2024-11-24T22:15:38,199 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/94f3ccf9f8a638fa974e1f69627832ef/C/644e239d0c2a44909d1456f6cd2cb2b2, entries=150, sequenceid=189, filesize=11.9 K 2024-11-24T22:15:38,200 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~73.80 KB/75570, heapSize ~219.84 KB/225120, currentSize=100.63 KB/103050 for 94f3ccf9f8a638fa974e1f69627832ef in 1285ms, sequenceid=189, compaction requested=false 2024-11-24T22:15:38,200 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 94f3ccf9f8a638fa974e1f69627832ef: 2024-11-24T22:15:38,227 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40087 {}] regionserver.HRegion(8855): Flush requested on 94f3ccf9f8a638fa974e1f69627832ef 2024-11-24T22:15:38,227 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 94f3ccf9f8a638fa974e1f69627832ef 3/3 column families, dataSize=107.34 KB heapSize=319.50 KB 2024-11-24T22:15:38,228 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 94f3ccf9f8a638fa974e1f69627832ef, store=A 2024-11-24T22:15:38,228 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-24T22:15:38,228 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 94f3ccf9f8a638fa974e1f69627832ef, store=B 2024-11-24T22:15:38,228 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-24T22:15:38,228 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 94f3ccf9f8a638fa974e1f69627832ef, store=C 2024-11-24T22:15:38,228 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-24T22:15:38,237 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241124fb1ff857056d430e8a2a87ac743e865a_94f3ccf9f8a638fa974e1f69627832ef is 50, key is test_row_0/A:col10/1732486537050/Put/seqid=0 2024-11-24T22:15:38,241 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45929 is added to blk_1073742236_1412 (size=14794) 2024-11-24T22:15:38,245 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T22:15:38,248 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241124fb1ff857056d430e8a2a87ac743e865a_94f3ccf9f8a638fa974e1f69627832ef to hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241124fb1ff857056d430e8a2a87ac743e865a_94f3ccf9f8a638fa974e1f69627832ef 2024-11-24T22:15:38,248 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/94f3ccf9f8a638fa974e1f69627832ef/.tmp/A/546c086e2e0c4208ba4af9d5985b3e9b, store: [table=TestAcidGuarantees family=A region=94f3ccf9f8a638fa974e1f69627832ef] 2024-11-24T22:15:38,249 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/94f3ccf9f8a638fa974e1f69627832ef/.tmp/A/546c086e2e0c4208ba4af9d5985b3e9b is 175, key is test_row_0/A:col10/1732486537050/Put/seqid=0 2024-11-24T22:15:38,253 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45929 is added to blk_1073742237_1413 (size=39749) 2024-11-24T22:15:38,253 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=211, memsize=35.8 K, hasBloomFilter=true, into tmp file hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/94f3ccf9f8a638fa974e1f69627832ef/.tmp/A/546c086e2e0c4208ba4af9d5985b3e9b 2024-11-24T22:15:38,272 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/94f3ccf9f8a638fa974e1f69627832ef/.tmp/B/39ded812f39945be83f4845513cfe2f2 is 50, key is test_row_0/B:col10/1732486537050/Put/seqid=0 2024-11-24T22:15:38,274 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40087 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94f3ccf9f8a638fa974e1f69627832ef, server=bd6592c37bb2,40087,1732486423713 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-24T22:15:38,274 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40087 {}] ipc.CallRunner(138): callId: 72 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:32774 deadline: 1732486598260, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94f3ccf9f8a638fa974e1f69627832ef, server=bd6592c37bb2,40087,1732486423713 2024-11-24T22:15:38,275 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40087 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94f3ccf9f8a638fa974e1f69627832ef, server=bd6592c37bb2,40087,1732486423713 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-24T22:15:38,275 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732486521610.94f3ccf9f8a638fa974e1f69627832ef., hostname=bd6592c37bb2,40087,1732486423713, seqNum=5 , the old value is region=TestAcidGuarantees,,1732486521610.94f3ccf9f8a638fa974e1f69627832ef., hostname=bd6592c37bb2,40087,1732486423713, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94f3ccf9f8a638fa974e1f69627832ef, server=bd6592c37bb2,40087,1732486423713 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T22:15:38,275 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732486521610.94f3ccf9f8a638fa974e1f69627832ef., hostname=bd6592c37bb2,40087,1732486423713, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94f3ccf9f8a638fa974e1f69627832ef, server=bd6592c37bb2,40087,1732486423713 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T22:15:38,275 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732486521610.94f3ccf9f8a638fa974e1f69627832ef., hostname=bd6592c37bb2,40087,1732486423713, seqNum=5 because the exception is null or not the one we care about 2024-11-24T22:15:38,275 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40087 {}] ipc.CallRunner(138): callId: 73 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:32846 deadline: 1732486598262, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94f3ccf9f8a638fa974e1f69627832ef, server=bd6592c37bb2,40087,1732486423713 2024-11-24T22:15:38,275 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732486521610.94f3ccf9f8a638fa974e1f69627832ef., hostname=bd6592c37bb2,40087,1732486423713, seqNum=5 , the old value is region=TestAcidGuarantees,,1732486521610.94f3ccf9f8a638fa974e1f69627832ef., hostname=bd6592c37bb2,40087,1732486423713, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94f3ccf9f8a638fa974e1f69627832ef, server=bd6592c37bb2,40087,1732486423713 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T22:15:38,275 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732486521610.94f3ccf9f8a638fa974e1f69627832ef., hostname=bd6592c37bb2,40087,1732486423713, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94f3ccf9f8a638fa974e1f69627832ef, server=bd6592c37bb2,40087,1732486423713 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T22:15:38,275 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732486521610.94f3ccf9f8a638fa974e1f69627832ef., hostname=bd6592c37bb2,40087,1732486423713, seqNum=5 because the exception is null or not the one we care about 2024-11-24T22:15:38,277 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=40087 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=104 2024-11-24T22:15:38,277 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45929 is added to blk_1073742238_1414 (size=12151) 2024-11-24T22:15:38,277 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bd6592c37bb2:0-1 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732486521610.94f3ccf9f8a638fa974e1f69627832ef. 2024-11-24T22:15:38,277 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bd6592c37bb2:0-1 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732486521610.94f3ccf9f8a638fa974e1f69627832ef. as already flushing 2024-11-24T22:15:38,277 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bd6592c37bb2:0-1 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732486521610.94f3ccf9f8a638fa974e1f69627832ef. 2024-11-24T22:15:38,277 ERROR [RS_FLUSH_OPERATIONS-regionserver/bd6592c37bb2:0-1 {event_type=RS_FLUSH_REGIONS, pid=104}] handler.RSProcedureHandler(58): pid=104 java.io.IOException: Unable to complete flush {ENCODED => 94f3ccf9f8a638fa974e1f69627832ef, NAME => 'TestAcidGuarantees,,1732486521610.94f3ccf9f8a638fa974e1f69627832ef.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T22:15:38,277 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bd6592c37bb2:0-1 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=104 java.io.IOException: Unable to complete flush {ENCODED => 94f3ccf9f8a638fa974e1f69627832ef, NAME => 'TestAcidGuarantees,,1732486521610.94f3ccf9f8a638fa974e1f69627832ef.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T22:15:38,278 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37701 {}] master.HMaster(4177): Remote procedure failed, pid=104 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 94f3ccf9f8a638fa974e1f69627832ef, NAME => 'TestAcidGuarantees,,1732486521610.94f3ccf9f8a638fa974e1f69627832ef.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 94f3ccf9f8a638fa974e1f69627832ef, NAME => 'TestAcidGuarantees,,1732486521610.94f3ccf9f8a638fa974e1f69627832ef.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T22:15:38,282 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40087 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94f3ccf9f8a638fa974e1f69627832ef, server=bd6592c37bb2,40087,1732486423713 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-24T22:15:38,283 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40087 {}] ipc.CallRunner(138): callId: 57 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:32832 deadline: 1732486598275, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94f3ccf9f8a638fa974e1f69627832ef, server=bd6592c37bb2,40087,1732486423713 2024-11-24T22:15:38,283 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732486521610.94f3ccf9f8a638fa974e1f69627832ef., hostname=bd6592c37bb2,40087,1732486423713, seqNum=5 , the old value is region=TestAcidGuarantees,,1732486521610.94f3ccf9f8a638fa974e1f69627832ef., hostname=bd6592c37bb2,40087,1732486423713, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94f3ccf9f8a638fa974e1f69627832ef, server=bd6592c37bb2,40087,1732486423713 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T22:15:38,283 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732486521610.94f3ccf9f8a638fa974e1f69627832ef., hostname=bd6592c37bb2,40087,1732486423713, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94f3ccf9f8a638fa974e1f69627832ef, server=bd6592c37bb2,40087,1732486423713 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T22:15:38,283 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732486521610.94f3ccf9f8a638fa974e1f69627832ef., hostname=bd6592c37bb2,40087,1732486423713, seqNum=5 because the exception is null or not the one we care about 2024-11-24T22:15:38,340 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37701 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=103 2024-11-24T22:15:38,390 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40087 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94f3ccf9f8a638fa974e1f69627832ef, server=bd6592c37bb2,40087,1732486423713 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-24T22:15:38,390 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40087 {}] ipc.CallRunner(138): callId: 73 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:32774 deadline: 1732486598381, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94f3ccf9f8a638fa974e1f69627832ef, server=bd6592c37bb2,40087,1732486423713 2024-11-24T22:15:38,390 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732486521610.94f3ccf9f8a638fa974e1f69627832ef., hostname=bd6592c37bb2,40087,1732486423713, seqNum=5 , the old value is region=TestAcidGuarantees,,1732486521610.94f3ccf9f8a638fa974e1f69627832ef., hostname=bd6592c37bb2,40087,1732486423713, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94f3ccf9f8a638fa974e1f69627832ef, server=bd6592c37bb2,40087,1732486423713 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T22:15:38,390 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40087 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94f3ccf9f8a638fa974e1f69627832ef, server=bd6592c37bb2,40087,1732486423713 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-24T22:15:38,390 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732486521610.94f3ccf9f8a638fa974e1f69627832ef., hostname=bd6592c37bb2,40087,1732486423713, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94f3ccf9f8a638fa974e1f69627832ef, server=bd6592c37bb2,40087,1732486423713 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T22:15:38,391 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732486521610.94f3ccf9f8a638fa974e1f69627832ef., hostname=bd6592c37bb2,40087,1732486423713, seqNum=5 because the exception is null or not the one we care about 2024-11-24T22:15:38,391 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40087 {}] ipc.CallRunner(138): callId: 74 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:32846 deadline: 1732486598381, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94f3ccf9f8a638fa974e1f69627832ef, server=bd6592c37bb2,40087,1732486423713 2024-11-24T22:15:38,391 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732486521610.94f3ccf9f8a638fa974e1f69627832ef., hostname=bd6592c37bb2,40087,1732486423713, seqNum=5 , the old value is region=TestAcidGuarantees,,1732486521610.94f3ccf9f8a638fa974e1f69627832ef., hostname=bd6592c37bb2,40087,1732486423713, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94f3ccf9f8a638fa974e1f69627832ef, server=bd6592c37bb2,40087,1732486423713 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T22:15:38,391 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732486521610.94f3ccf9f8a638fa974e1f69627832ef., hostname=bd6592c37bb2,40087,1732486423713, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94f3ccf9f8a638fa974e1f69627832ef, server=bd6592c37bb2,40087,1732486423713 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T22:15:38,391 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732486521610.94f3ccf9f8a638fa974e1f69627832ef., hostname=bd6592c37bb2,40087,1732486423713, seqNum=5 because the exception is null or not the one we care about 2024-11-24T22:15:38,400 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40087 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94f3ccf9f8a638fa974e1f69627832ef, server=bd6592c37bb2,40087,1732486423713 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-24T22:15:38,400 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40087 {}] ipc.CallRunner(138): callId: 58 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:32832 deadline: 1732486598391, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94f3ccf9f8a638fa974e1f69627832ef, server=bd6592c37bb2,40087,1732486423713 2024-11-24T22:15:38,400 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732486521610.94f3ccf9f8a638fa974e1f69627832ef., hostname=bd6592c37bb2,40087,1732486423713, seqNum=5 , the old value is region=TestAcidGuarantees,,1732486521610.94f3ccf9f8a638fa974e1f69627832ef., hostname=bd6592c37bb2,40087,1732486423713, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94f3ccf9f8a638fa974e1f69627832ef, server=bd6592c37bb2,40087,1732486423713 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T22:15:38,400 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732486521610.94f3ccf9f8a638fa974e1f69627832ef., hostname=bd6592c37bb2,40087,1732486423713, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94f3ccf9f8a638fa974e1f69627832ef, server=bd6592c37bb2,40087,1732486423713 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T22:15:38,400 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732486521610.94f3ccf9f8a638fa974e1f69627832ef., hostname=bd6592c37bb2,40087,1732486423713, seqNum=5 because the exception is null or not the one we care about 2024-11-24T22:15:38,429 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=40087 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=104 2024-11-24T22:15:38,429 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bd6592c37bb2:0-2 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732486521610.94f3ccf9f8a638fa974e1f69627832ef. 2024-11-24T22:15:38,430 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bd6592c37bb2:0-2 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732486521610.94f3ccf9f8a638fa974e1f69627832ef. as already flushing 2024-11-24T22:15:38,430 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bd6592c37bb2:0-2 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732486521610.94f3ccf9f8a638fa974e1f69627832ef. 2024-11-24T22:15:38,430 ERROR [RS_FLUSH_OPERATIONS-regionserver/bd6592c37bb2:0-2 {event_type=RS_FLUSH_REGIONS, pid=104}] handler.RSProcedureHandler(58): pid=104 java.io.IOException: Unable to complete flush {ENCODED => 94f3ccf9f8a638fa974e1f69627832ef, NAME => 'TestAcidGuarantees,,1732486521610.94f3ccf9f8a638fa974e1f69627832ef.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T22:15:38,430 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bd6592c37bb2:0-2 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=104 java.io.IOException: Unable to complete flush {ENCODED => 94f3ccf9f8a638fa974e1f69627832ef, NAME => 'TestAcidGuarantees,,1732486521610.94f3ccf9f8a638fa974e1f69627832ef.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T22:15:38,431 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37701 {}] master.HMaster(4177): Remote procedure failed, pid=104 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 94f3ccf9f8a638fa974e1f69627832ef, NAME => 'TestAcidGuarantees,,1732486521610.94f3ccf9f8a638fa974e1f69627832ef.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 94f3ccf9f8a638fa974e1f69627832ef, NAME => 'TestAcidGuarantees,,1732486521610.94f3ccf9f8a638fa974e1f69627832ef.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T22:15:38,582 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=40087 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=104 2024-11-24T22:15:38,583 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bd6592c37bb2:0-0 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732486521610.94f3ccf9f8a638fa974e1f69627832ef. 2024-11-24T22:15:38,583 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bd6592c37bb2:0-0 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732486521610.94f3ccf9f8a638fa974e1f69627832ef. as already flushing 2024-11-24T22:15:38,583 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bd6592c37bb2:0-0 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732486521610.94f3ccf9f8a638fa974e1f69627832ef. 2024-11-24T22:15:38,583 ERROR [RS_FLUSH_OPERATIONS-regionserver/bd6592c37bb2:0-0 {event_type=RS_FLUSH_REGIONS, pid=104}] handler.RSProcedureHandler(58): pid=104 java.io.IOException: Unable to complete flush {ENCODED => 94f3ccf9f8a638fa974e1f69627832ef, NAME => 'TestAcidGuarantees,,1732486521610.94f3ccf9f8a638fa974e1f69627832ef.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T22:15:38,583 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bd6592c37bb2:0-0 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=104 java.io.IOException: Unable to complete flush {ENCODED => 94f3ccf9f8a638fa974e1f69627832ef, NAME => 'TestAcidGuarantees,,1732486521610.94f3ccf9f8a638fa974e1f69627832ef.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T22:15:38,583 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37701 {}] master.HMaster(4177): Remote procedure failed, pid=104 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 94f3ccf9f8a638fa974e1f69627832ef, NAME => 'TestAcidGuarantees,,1732486521610.94f3ccf9f8a638fa974e1f69627832ef.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 94f3ccf9f8a638fa974e1f69627832ef, NAME => 'TestAcidGuarantees,,1732486521610.94f3ccf9f8a638fa974e1f69627832ef.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T22:15:38,612 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40087 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94f3ccf9f8a638fa974e1f69627832ef, server=bd6592c37bb2,40087,1732486423713 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-24T22:15:38,612 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40087 {}] ipc.CallRunner(138): callId: 74 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:32774 deadline: 1732486598600, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94f3ccf9f8a638fa974e1f69627832ef, server=bd6592c37bb2,40087,1732486423713 2024-11-24T22:15:38,612 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40087 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94f3ccf9f8a638fa974e1f69627832ef, server=bd6592c37bb2,40087,1732486423713 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-24T22:15:38,612 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732486521610.94f3ccf9f8a638fa974e1f69627832ef., hostname=bd6592c37bb2,40087,1732486423713, seqNum=5 , the old value is region=TestAcidGuarantees,,1732486521610.94f3ccf9f8a638fa974e1f69627832ef., hostname=bd6592c37bb2,40087,1732486423713, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94f3ccf9f8a638fa974e1f69627832ef, server=bd6592c37bb2,40087,1732486423713 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T22:15:38,613 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732486521610.94f3ccf9f8a638fa974e1f69627832ef., hostname=bd6592c37bb2,40087,1732486423713, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94f3ccf9f8a638fa974e1f69627832ef, server=bd6592c37bb2,40087,1732486423713 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T22:15:38,613 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40087 {}] ipc.CallRunner(138): callId: 75 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:32846 deadline: 1732486598601, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94f3ccf9f8a638fa974e1f69627832ef, server=bd6592c37bb2,40087,1732486423713 2024-11-24T22:15:38,613 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732486521610.94f3ccf9f8a638fa974e1f69627832ef., hostname=bd6592c37bb2,40087,1732486423713, seqNum=5 because the exception is null or not the one we care about 2024-11-24T22:15:38,613 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732486521610.94f3ccf9f8a638fa974e1f69627832ef., hostname=bd6592c37bb2,40087,1732486423713, seqNum=5 , the old value is region=TestAcidGuarantees,,1732486521610.94f3ccf9f8a638fa974e1f69627832ef., hostname=bd6592c37bb2,40087,1732486423713, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94f3ccf9f8a638fa974e1f69627832ef, server=bd6592c37bb2,40087,1732486423713 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T22:15:38,613 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732486521610.94f3ccf9f8a638fa974e1f69627832ef., hostname=bd6592c37bb2,40087,1732486423713, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94f3ccf9f8a638fa974e1f69627832ef, server=bd6592c37bb2,40087,1732486423713 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T22:15:38,613 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732486521610.94f3ccf9f8a638fa974e1f69627832ef., hostname=bd6592c37bb2,40087,1732486423713, seqNum=5 because the exception is null or not the one we care about 2024-11-24T22:15:38,623 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40087 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94f3ccf9f8a638fa974e1f69627832ef, server=bd6592c37bb2,40087,1732486423713 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-24T22:15:38,623 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40087 {}] ipc.CallRunner(138): callId: 59 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:32832 deadline: 1732486598611, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94f3ccf9f8a638fa974e1f69627832ef, server=bd6592c37bb2,40087,1732486423713 2024-11-24T22:15:38,623 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732486521610.94f3ccf9f8a638fa974e1f69627832ef., hostname=bd6592c37bb2,40087,1732486423713, seqNum=5 , the old value is region=TestAcidGuarantees,,1732486521610.94f3ccf9f8a638fa974e1f69627832ef., hostname=bd6592c37bb2,40087,1732486423713, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94f3ccf9f8a638fa974e1f69627832ef, server=bd6592c37bb2,40087,1732486423713 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T22:15:38,623 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732486521610.94f3ccf9f8a638fa974e1f69627832ef., hostname=bd6592c37bb2,40087,1732486423713, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94f3ccf9f8a638fa974e1f69627832ef, server=bd6592c37bb2,40087,1732486423713 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T22:15:38,623 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732486521610.94f3ccf9f8a638fa974e1f69627832ef., hostname=bd6592c37bb2,40087,1732486423713, seqNum=5 because the exception is null or not the one we care about 2024-11-24T22:15:38,677 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=35.78 KB at sequenceid=211 (bloomFilter=true), to=hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/94f3ccf9f8a638fa974e1f69627832ef/.tmp/B/39ded812f39945be83f4845513cfe2f2 2024-11-24T22:15:38,684 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/94f3ccf9f8a638fa974e1f69627832ef/.tmp/C/1f70e3bbb0e24ae8b788a7fc0abb2288 is 50, key is test_row_0/C:col10/1732486537050/Put/seqid=0 2024-11-24T22:15:38,691 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45929 is added to blk_1073742239_1415 (size=12151) 2024-11-24T22:15:38,735 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=40087 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=104 2024-11-24T22:15:38,735 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bd6592c37bb2:0-1 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732486521610.94f3ccf9f8a638fa974e1f69627832ef. 2024-11-24T22:15:38,736 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bd6592c37bb2:0-1 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732486521610.94f3ccf9f8a638fa974e1f69627832ef. as already flushing 2024-11-24T22:15:38,736 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bd6592c37bb2:0-1 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732486521610.94f3ccf9f8a638fa974e1f69627832ef. 2024-11-24T22:15:38,736 ERROR [RS_FLUSH_OPERATIONS-regionserver/bd6592c37bb2:0-1 {event_type=RS_FLUSH_REGIONS, pid=104}] handler.RSProcedureHandler(58): pid=104 java.io.IOException: Unable to complete flush {ENCODED => 94f3ccf9f8a638fa974e1f69627832ef, NAME => 'TestAcidGuarantees,,1732486521610.94f3ccf9f8a638fa974e1f69627832ef.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T22:15:38,736 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bd6592c37bb2:0-1 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=104 java.io.IOException: Unable to complete flush {ENCODED => 94f3ccf9f8a638fa974e1f69627832ef, NAME => 'TestAcidGuarantees,,1732486521610.94f3ccf9f8a638fa974e1f69627832ef.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T22:15:38,736 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37701 {}] master.HMaster(4177): Remote procedure failed, pid=104 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 94f3ccf9f8a638fa974e1f69627832ef, NAME => 'TestAcidGuarantees,,1732486521610.94f3ccf9f8a638fa974e1f69627832ef.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 94f3ccf9f8a638fa974e1f69627832ef, NAME => 'TestAcidGuarantees,,1732486521610.94f3ccf9f8a638fa974e1f69627832ef.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T22:15:38,888 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=40087 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=104 2024-11-24T22:15:38,889 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bd6592c37bb2:0-2 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732486521610.94f3ccf9f8a638fa974e1f69627832ef. 2024-11-24T22:15:38,889 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bd6592c37bb2:0-2 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732486521610.94f3ccf9f8a638fa974e1f69627832ef. as already flushing 2024-11-24T22:15:38,889 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bd6592c37bb2:0-2 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732486521610.94f3ccf9f8a638fa974e1f69627832ef. 2024-11-24T22:15:38,889 ERROR [RS_FLUSH_OPERATIONS-regionserver/bd6592c37bb2:0-2 {event_type=RS_FLUSH_REGIONS, pid=104}] handler.RSProcedureHandler(58): pid=104 java.io.IOException: Unable to complete flush {ENCODED => 94f3ccf9f8a638fa974e1f69627832ef, NAME => 'TestAcidGuarantees,,1732486521610.94f3ccf9f8a638fa974e1f69627832ef.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T22:15:38,889 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bd6592c37bb2:0-2 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=104 java.io.IOException: Unable to complete flush {ENCODED => 94f3ccf9f8a638fa974e1f69627832ef, NAME => 'TestAcidGuarantees,,1732486521610.94f3ccf9f8a638fa974e1f69627832ef.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T22:15:38,889 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37701 {}] master.HMaster(4177): Remote procedure failed, pid=104 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 94f3ccf9f8a638fa974e1f69627832ef, NAME => 'TestAcidGuarantees,,1732486521610.94f3ccf9f8a638fa974e1f69627832ef.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 94f3ccf9f8a638fa974e1f69627832ef, NAME => 'TestAcidGuarantees,,1732486521610.94f3ccf9f8a638fa974e1f69627832ef.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T22:15:38,930 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40087 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94f3ccf9f8a638fa974e1f69627832ef, server=bd6592c37bb2,40087,1732486423713 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-24T22:15:38,930 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40087 {}] ipc.CallRunner(138): callId: 75 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:32774 deadline: 1732486598921, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94f3ccf9f8a638fa974e1f69627832ef, server=bd6592c37bb2,40087,1732486423713 2024-11-24T22:15:38,930 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732486521610.94f3ccf9f8a638fa974e1f69627832ef., hostname=bd6592c37bb2,40087,1732486423713, seqNum=5 , the old value is region=TestAcidGuarantees,,1732486521610.94f3ccf9f8a638fa974e1f69627832ef., hostname=bd6592c37bb2,40087,1732486423713, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94f3ccf9f8a638fa974e1f69627832ef, server=bd6592c37bb2,40087,1732486423713 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T22:15:38,931 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732486521610.94f3ccf9f8a638fa974e1f69627832ef., hostname=bd6592c37bb2,40087,1732486423713, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94f3ccf9f8a638fa974e1f69627832ef, server=bd6592c37bb2,40087,1732486423713 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T22:15:38,931 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732486521610.94f3ccf9f8a638fa974e1f69627832ef., hostname=bd6592c37bb2,40087,1732486423713, seqNum=5 because the exception is null or not the one we care about 2024-11-24T22:15:38,930 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40087 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94f3ccf9f8a638fa974e1f69627832ef, server=bd6592c37bb2,40087,1732486423713 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-24T22:15:38,931 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40087 {}] ipc.CallRunner(138): callId: 76 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:32846 deadline: 1732486598921, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94f3ccf9f8a638fa974e1f69627832ef, server=bd6592c37bb2,40087,1732486423713 2024-11-24T22:15:38,931 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732486521610.94f3ccf9f8a638fa974e1f69627832ef., hostname=bd6592c37bb2,40087,1732486423713, seqNum=5 , the old value is region=TestAcidGuarantees,,1732486521610.94f3ccf9f8a638fa974e1f69627832ef., hostname=bd6592c37bb2,40087,1732486423713, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94f3ccf9f8a638fa974e1f69627832ef, server=bd6592c37bb2,40087,1732486423713 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T22:15:38,931 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732486521610.94f3ccf9f8a638fa974e1f69627832ef., hostname=bd6592c37bb2,40087,1732486423713, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94f3ccf9f8a638fa974e1f69627832ef, server=bd6592c37bb2,40087,1732486423713 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T22:15:38,931 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732486521610.94f3ccf9f8a638fa974e1f69627832ef., hostname=bd6592c37bb2,40087,1732486423713, seqNum=5 because the exception is null or not the one we care about 2024-11-24T22:15:38,941 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40087 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94f3ccf9f8a638fa974e1f69627832ef, server=bd6592c37bb2,40087,1732486423713 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-24T22:15:38,941 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40087 {}] ipc.CallRunner(138): callId: 60 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:32832 deadline: 1732486598931, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94f3ccf9f8a638fa974e1f69627832ef, server=bd6592c37bb2,40087,1732486423713 2024-11-24T22:15:38,941 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40087 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94f3ccf9f8a638fa974e1f69627832ef, server=bd6592c37bb2,40087,1732486423713 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-24T22:15:38,941 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40087 {}] ipc.CallRunner(138): callId: 61 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:32810 deadline: 1732486598931, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94f3ccf9f8a638fa974e1f69627832ef, server=bd6592c37bb2,40087,1732486423713 2024-11-24T22:15:38,941 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732486521610.94f3ccf9f8a638fa974e1f69627832ef., hostname=bd6592c37bb2,40087,1732486423713, seqNum=5 , the old value is region=TestAcidGuarantees,,1732486521610.94f3ccf9f8a638fa974e1f69627832ef., hostname=bd6592c37bb2,40087,1732486423713, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94f3ccf9f8a638fa974e1f69627832ef, server=bd6592c37bb2,40087,1732486423713 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T22:15:38,941 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732486521610.94f3ccf9f8a638fa974e1f69627832ef., hostname=bd6592c37bb2,40087,1732486423713, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94f3ccf9f8a638fa974e1f69627832ef, server=bd6592c37bb2,40087,1732486423713 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T22:15:38,941 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732486521610.94f3ccf9f8a638fa974e1f69627832ef., hostname=bd6592c37bb2,40087,1732486423713, seqNum=5 because the exception is null or not the one we care about 2024-11-24T22:15:38,942 WARN [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRpcRetryingCaller(177): Call to bd6592c37bb2,40087,1732486423713 for 'test_row_2' in 94f3ccf9f8a638fa974e1f69627832ef of TestAcidGuarantees failed, tries = 7, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 4219 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94f3ccf9f8a638fa974e1f69627832ef, server=bd6592c37bb2,40087,1732486423713 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor44.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94f3ccf9f8a638fa974e1f69627832ef, server=bd6592c37bb2,40087,1732486423713 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-24T22:15:38,942 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732486521610.94f3ccf9f8a638fa974e1f69627832ef., hostname=bd6592c37bb2,40087,1732486423713, seqNum=5 , the old value is region=TestAcidGuarantees,,1732486521610.94f3ccf9f8a638fa974e1f69627832ef., hostname=bd6592c37bb2,40087,1732486423713, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94f3ccf9f8a638fa974e1f69627832ef, server=bd6592c37bb2,40087,1732486423713 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T22:15:38,942 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732486521610.94f3ccf9f8a638fa974e1f69627832ef., hostname=bd6592c37bb2,40087,1732486423713, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94f3ccf9f8a638fa974e1f69627832ef, server=bd6592c37bb2,40087,1732486423713 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T22:15:38,942 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732486521610.94f3ccf9f8a638fa974e1f69627832ef., hostname=bd6592c37bb2,40087,1732486423713, seqNum=5 because the exception is null or not the one we care about 2024-11-24T22:15:39,041 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=40087 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=104 2024-11-24T22:15:39,041 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bd6592c37bb2:0-0 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732486521610.94f3ccf9f8a638fa974e1f69627832ef. 2024-11-24T22:15:39,042 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bd6592c37bb2:0-0 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732486521610.94f3ccf9f8a638fa974e1f69627832ef. as already flushing 2024-11-24T22:15:39,042 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bd6592c37bb2:0-0 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732486521610.94f3ccf9f8a638fa974e1f69627832ef. 2024-11-24T22:15:39,042 ERROR [RS_FLUSH_OPERATIONS-regionserver/bd6592c37bb2:0-0 {event_type=RS_FLUSH_REGIONS, pid=104}] handler.RSProcedureHandler(58): pid=104 java.io.IOException: Unable to complete flush {ENCODED => 94f3ccf9f8a638fa974e1f69627832ef, NAME => 'TestAcidGuarantees,,1732486521610.94f3ccf9f8a638fa974e1f69627832ef.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T22:15:39,042 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bd6592c37bb2:0-0 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=104 java.io.IOException: Unable to complete flush {ENCODED => 94f3ccf9f8a638fa974e1f69627832ef, NAME => 'TestAcidGuarantees,,1732486521610.94f3ccf9f8a638fa974e1f69627832ef.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T22:15:39,042 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37701 {}] master.HMaster(4177): Remote procedure failed, pid=104 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 94f3ccf9f8a638fa974e1f69627832ef, NAME => 'TestAcidGuarantees,,1732486521610.94f3ccf9f8a638fa974e1f69627832ef.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 94f3ccf9f8a638fa974e1f69627832ef, NAME => 'TestAcidGuarantees,,1732486521610.94f3ccf9f8a638fa974e1f69627832ef.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T22:15:39,091 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=35.78 KB at sequenceid=211 (bloomFilter=true), to=hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/94f3ccf9f8a638fa974e1f69627832ef/.tmp/C/1f70e3bbb0e24ae8b788a7fc0abb2288 2024-11-24T22:15:39,095 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/94f3ccf9f8a638fa974e1f69627832ef/.tmp/A/546c086e2e0c4208ba4af9d5985b3e9b as hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/94f3ccf9f8a638fa974e1f69627832ef/A/546c086e2e0c4208ba4af9d5985b3e9b 2024-11-24T22:15:39,098 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/94f3ccf9f8a638fa974e1f69627832ef/A/546c086e2e0c4208ba4af9d5985b3e9b, entries=200, sequenceid=211, filesize=38.8 K 2024-11-24T22:15:39,099 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/94f3ccf9f8a638fa974e1f69627832ef/.tmp/B/39ded812f39945be83f4845513cfe2f2 as hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/94f3ccf9f8a638fa974e1f69627832ef/B/39ded812f39945be83f4845513cfe2f2 2024-11-24T22:15:39,102 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/94f3ccf9f8a638fa974e1f69627832ef/B/39ded812f39945be83f4845513cfe2f2, entries=150, sequenceid=211, filesize=11.9 K 2024-11-24T22:15:39,102 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/94f3ccf9f8a638fa974e1f69627832ef/.tmp/C/1f70e3bbb0e24ae8b788a7fc0abb2288 as hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/94f3ccf9f8a638fa974e1f69627832ef/C/1f70e3bbb0e24ae8b788a7fc0abb2288 2024-11-24T22:15:39,105 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/94f3ccf9f8a638fa974e1f69627832ef/C/1f70e3bbb0e24ae8b788a7fc0abb2288, entries=150, sequenceid=211, filesize=11.9 K 2024-11-24T22:15:39,105 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~107.34 KB/109920, heapSize ~319.45 KB/327120, currentSize=67.09 KB/68700 for 94f3ccf9f8a638fa974e1f69627832ef in 878ms, sequenceid=211, compaction requested=true 2024-11-24T22:15:39,106 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 94f3ccf9f8a638fa974e1f69627832ef: 2024-11-24T22:15:39,106 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 94f3ccf9f8a638fa974e1f69627832ef:A, priority=-2147483648, current under compaction store size is 1 2024-11-24T22:15:39,106 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-24T22:15:39,106 DEBUG [RS:0;bd6592c37bb2:40087-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-24T22:15:39,106 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 94f3ccf9f8a638fa974e1f69627832ef:B, priority=-2147483648, current under compaction store size is 2 2024-11-24T22:15:39,106 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-24T22:15:39,106 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 94f3ccf9f8a638fa974e1f69627832ef:C, priority=-2147483648, current under compaction store size is 3 2024-11-24T22:15:39,106 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-24T22:15:39,106 DEBUG [RS:0;bd6592c37bb2:40087-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-24T22:15:39,107 DEBUG [RS:0;bd6592c37bb2:40087-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 111047 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-24T22:15:39,107 DEBUG [RS:0;bd6592c37bb2:40087-shortCompactions-0 {}] regionserver.HStore(1541): 94f3ccf9f8a638fa974e1f69627832ef/A is initiating minor compaction (all files) 2024-11-24T22:15:39,107 INFO [RS:0;bd6592c37bb2:40087-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 94f3ccf9f8a638fa974e1f69627832ef/A in TestAcidGuarantees,,1732486521610.94f3ccf9f8a638fa974e1f69627832ef. 2024-11-24T22:15:39,107 INFO [RS:0;bd6592c37bb2:40087-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/94f3ccf9f8a638fa974e1f69627832ef/A/17c6ecafbb0d4ff6b176abbe1aed6b68, hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/94f3ccf9f8a638fa974e1f69627832ef/A/28d65c9086ac4081b22ae0ce11933e15, hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/94f3ccf9f8a638fa974e1f69627832ef/A/546c086e2e0c4208ba4af9d5985b3e9b] into tmpdir=hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/94f3ccf9f8a638fa974e1f69627832ef/.tmp, totalSize=108.4 K 2024-11-24T22:15:39,107 INFO [RS:0;bd6592c37bb2:40087-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=13 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1732486521610.94f3ccf9f8a638fa974e1f69627832ef. 2024-11-24T22:15:39,107 DEBUG [RS:0;bd6592c37bb2:40087-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1732486521610.94f3ccf9f8a638fa974e1f69627832ef. files: [hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/94f3ccf9f8a638fa974e1f69627832ef/A/17c6ecafbb0d4ff6b176abbe1aed6b68, hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/94f3ccf9f8a638fa974e1f69627832ef/A/28d65c9086ac4081b22ae0ce11933e15, hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/94f3ccf9f8a638fa974e1f69627832ef/A/546c086e2e0c4208ba4af9d5985b3e9b] 2024-11-24T22:15:39,108 DEBUG [RS:0;bd6592c37bb2:40087-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36897 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-24T22:15:39,108 DEBUG [RS:0;bd6592c37bb2:40087-shortCompactions-0 {}] compactions.Compactor(225): Compacting 17c6ecafbb0d4ff6b176abbe1aed6b68, keycount=150, bloomtype=ROW, size=30.8 K, encoding=NONE, compression=NONE, seqNum=175, earliestPutTs=1732486534720 2024-11-24T22:15:39,108 DEBUG [RS:0;bd6592c37bb2:40087-longCompactions-0 {}] regionserver.HStore(1541): 94f3ccf9f8a638fa974e1f69627832ef/B is initiating minor compaction (all files) 2024-11-24T22:15:39,108 INFO [RS:0;bd6592c37bb2:40087-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 94f3ccf9f8a638fa974e1f69627832ef/B in TestAcidGuarantees,,1732486521610.94f3ccf9f8a638fa974e1f69627832ef. 2024-11-24T22:15:39,108 INFO [RS:0;bd6592c37bb2:40087-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/94f3ccf9f8a638fa974e1f69627832ef/B/bb9965a6ef4f4109a98514be227d61cf, hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/94f3ccf9f8a638fa974e1f69627832ef/B/72127362fbeb4a0598f8132387bbe316, hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/94f3ccf9f8a638fa974e1f69627832ef/B/39ded812f39945be83f4845513cfe2f2] into tmpdir=hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/94f3ccf9f8a638fa974e1f69627832ef/.tmp, totalSize=36.0 K 2024-11-24T22:15:39,108 DEBUG [RS:0;bd6592c37bb2:40087-shortCompactions-0 {}] compactions.Compactor(225): Compacting 28d65c9086ac4081b22ae0ce11933e15, keycount=200, bloomtype=ROW, size=38.8 K, encoding=NONE, compression=NONE, seqNum=189, earliestPutTs=1732486535689 2024-11-24T22:15:39,108 DEBUG [RS:0;bd6592c37bb2:40087-longCompactions-0 {}] compactions.Compactor(225): Compacting bb9965a6ef4f4109a98514be227d61cf, keycount=150, bloomtype=ROW, size=12.3 K, encoding=NONE, compression=NONE, seqNum=175, earliestPutTs=1732486534720 2024-11-24T22:15:39,108 DEBUG [RS:0;bd6592c37bb2:40087-shortCompactions-0 {}] compactions.Compactor(225): Compacting 546c086e2e0c4208ba4af9d5985b3e9b, keycount=200, bloomtype=ROW, size=38.8 K, encoding=NONE, compression=NONE, seqNum=211, earliestPutTs=1732486537032 2024-11-24T22:15:39,108 DEBUG [RS:0;bd6592c37bb2:40087-longCompactions-0 {}] compactions.Compactor(225): Compacting 72127362fbeb4a0598f8132387bbe316, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=189, earliestPutTs=1732486535727 2024-11-24T22:15:39,109 DEBUG [RS:0;bd6592c37bb2:40087-longCompactions-0 {}] compactions.Compactor(225): Compacting 39ded812f39945be83f4845513cfe2f2, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=211, earliestPutTs=1732486537048 2024-11-24T22:15:39,113 INFO [RS:0;bd6592c37bb2:40087-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=94f3ccf9f8a638fa974e1f69627832ef] 2024-11-24T22:15:39,115 DEBUG [RS:0;bd6592c37bb2:40087-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e2024112488c83aef983d4faf8d6808e30a8f2147_94f3ccf9f8a638fa974e1f69627832ef store=[table=TestAcidGuarantees family=A region=94f3ccf9f8a638fa974e1f69627832ef] 2024-11-24T22:15:39,115 INFO [RS:0;bd6592c37bb2:40087-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 94f3ccf9f8a638fa974e1f69627832ef#B#compaction#348 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-24T22:15:39,116 DEBUG [RS:0;bd6592c37bb2:40087-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/94f3ccf9f8a638fa974e1f69627832ef/.tmp/B/3ef534ba6992421fb44a53ef9a12e571 is 50, key is test_row_0/B:col10/1732486537050/Put/seqid=0 2024-11-24T22:15:39,117 DEBUG [RS:0;bd6592c37bb2:40087-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e2024112488c83aef983d4faf8d6808e30a8f2147_94f3ccf9f8a638fa974e1f69627832ef, store=[table=TestAcidGuarantees family=A region=94f3ccf9f8a638fa974e1f69627832ef] 2024-11-24T22:15:39,117 DEBUG [RS:0;bd6592c37bb2:40087-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024112488c83aef983d4faf8d6808e30a8f2147_94f3ccf9f8a638fa974e1f69627832ef because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=94f3ccf9f8a638fa974e1f69627832ef] 2024-11-24T22:15:39,121 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45929 is added to blk_1073742241_1417 (size=4469) 2024-11-24T22:15:39,122 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45929 is added to blk_1073742240_1416 (size=12697) 2024-11-24T22:15:39,194 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=40087 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=104 2024-11-24T22:15:39,194 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bd6592c37bb2:0-1 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732486521610.94f3ccf9f8a638fa974e1f69627832ef. 2024-11-24T22:15:39,194 INFO [RS_FLUSH_OPERATIONS-regionserver/bd6592c37bb2:0-1 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.HRegion(2902): Flushing 94f3ccf9f8a638fa974e1f69627832ef 3/3 column families, dataSize=67.09 KB heapSize=199.97 KB 2024-11-24T22:15:39,195 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bd6592c37bb2:0-1 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 94f3ccf9f8a638fa974e1f69627832ef, store=A 2024-11-24T22:15:39,195 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bd6592c37bb2:0-1 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-24T22:15:39,195 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bd6592c37bb2:0-1 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 94f3ccf9f8a638fa974e1f69627832ef, store=B 2024-11-24T22:15:39,195 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bd6592c37bb2:0-1 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-24T22:15:39,195 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bd6592c37bb2:0-1 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 94f3ccf9f8a638fa974e1f69627832ef, store=C 2024-11-24T22:15:39,195 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bd6592c37bb2:0-1 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-24T22:15:39,201 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bd6592c37bb2:0-1 {event_type=RS_FLUSH_REGIONS, pid=104}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411245775428d17684f829ea4eb940d9c82dd_94f3ccf9f8a638fa974e1f69627832ef is 50, key is test_row_0/A:col10/1732486538260/Put/seqid=0 2024-11-24T22:15:39,204 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45929 is added to blk_1073742242_1418 (size=12304) 2024-11-24T22:15:39,350 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37701 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=103 2024-11-24T22:15:39,446 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40087 {}] regionserver.HRegion(8855): Flush requested on 94f3ccf9f8a638fa974e1f69627832ef 2024-11-24T22:15:39,446 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732486521610.94f3ccf9f8a638fa974e1f69627832ef. as already flushing 2024-11-24T22:15:39,511 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40087 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94f3ccf9f8a638fa974e1f69627832ef, server=bd6592c37bb2,40087,1732486423713 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-24T22:15:39,512 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40087 {}] ipc.CallRunner(138): callId: 65 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:32832 deadline: 1732486599497, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94f3ccf9f8a638fa974e1f69627832ef, server=bd6592c37bb2,40087,1732486423713 2024-11-24T22:15:39,512 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40087 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94f3ccf9f8a638fa974e1f69627832ef, server=bd6592c37bb2,40087,1732486423713 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-24T22:15:39,512 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732486521610.94f3ccf9f8a638fa974e1f69627832ef., hostname=bd6592c37bb2,40087,1732486423713, seqNum=5 , the old value is region=TestAcidGuarantees,,1732486521610.94f3ccf9f8a638fa974e1f69627832ef., hostname=bd6592c37bb2,40087,1732486423713, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94f3ccf9f8a638fa974e1f69627832ef, server=bd6592c37bb2,40087,1732486423713 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T22:15:39,512 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732486521610.94f3ccf9f8a638fa974e1f69627832ef., hostname=bd6592c37bb2,40087,1732486423713, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94f3ccf9f8a638fa974e1f69627832ef, server=bd6592c37bb2,40087,1732486423713 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T22:15:39,512 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732486521610.94f3ccf9f8a638fa974e1f69627832ef., hostname=bd6592c37bb2,40087,1732486423713, seqNum=5 because the exception is null or not the one we care about 2024-11-24T22:15:39,512 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40087 {}] ipc.CallRunner(138): callId: 82 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:32774 deadline: 1732486599502, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94f3ccf9f8a638fa974e1f69627832ef, server=bd6592c37bb2,40087,1732486423713 2024-11-24T22:15:39,512 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732486521610.94f3ccf9f8a638fa974e1f69627832ef., hostname=bd6592c37bb2,40087,1732486423713, seqNum=5 , the old value is region=TestAcidGuarantees,,1732486521610.94f3ccf9f8a638fa974e1f69627832ef., hostname=bd6592c37bb2,40087,1732486423713, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94f3ccf9f8a638fa974e1f69627832ef, server=bd6592c37bb2,40087,1732486423713 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T22:15:39,512 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732486521610.94f3ccf9f8a638fa974e1f69627832ef., hostname=bd6592c37bb2,40087,1732486423713, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94f3ccf9f8a638fa974e1f69627832ef, server=bd6592c37bb2,40087,1732486423713 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T22:15:39,512 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732486521610.94f3ccf9f8a638fa974e1f69627832ef., hostname=bd6592c37bb2,40087,1732486423713, seqNum=5 because the exception is null or not the one we care about 2024-11-24T22:15:39,521 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40087 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94f3ccf9f8a638fa974e1f69627832ef, server=bd6592c37bb2,40087,1732486423713 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-24T22:15:39,521 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40087 {}] ipc.CallRunner(138): callId: 83 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:32846 deadline: 1732486599504, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94f3ccf9f8a638fa974e1f69627832ef, server=bd6592c37bb2,40087,1732486423713 2024-11-24T22:15:39,522 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732486521610.94f3ccf9f8a638fa974e1f69627832ef., hostname=bd6592c37bb2,40087,1732486423713, seqNum=5 , the old value is region=TestAcidGuarantees,,1732486521610.94f3ccf9f8a638fa974e1f69627832ef., hostname=bd6592c37bb2,40087,1732486423713, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94f3ccf9f8a638fa974e1f69627832ef, server=bd6592c37bb2,40087,1732486423713 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T22:15:39,522 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732486521610.94f3ccf9f8a638fa974e1f69627832ef., hostname=bd6592c37bb2,40087,1732486423713, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94f3ccf9f8a638fa974e1f69627832ef, server=bd6592c37bb2,40087,1732486423713 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T22:15:39,522 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732486521610.94f3ccf9f8a638fa974e1f69627832ef., hostname=bd6592c37bb2,40087,1732486423713, seqNum=5 because the exception is null or not the one we care about 2024-11-24T22:15:39,522 INFO [RS:0;bd6592c37bb2:40087-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 94f3ccf9f8a638fa974e1f69627832ef#A#compaction#347 average throughput is 0.06 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-24T22:15:39,523 DEBUG [RS:0;bd6592c37bb2:40087-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/94f3ccf9f8a638fa974e1f69627832ef/.tmp/A/090547ea69e9495fb9b887db92c6c3da is 175, key is test_row_0/A:col10/1732486537050/Put/seqid=0 2024-11-24T22:15:39,526 DEBUG [RS:0;bd6592c37bb2:40087-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/94f3ccf9f8a638fa974e1f69627832ef/.tmp/B/3ef534ba6992421fb44a53ef9a12e571 as hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/94f3ccf9f8a638fa974e1f69627832ef/B/3ef534ba6992421fb44a53ef9a12e571 2024-11-24T22:15:39,527 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45929 is added to blk_1073742243_1419 (size=31651) 2024-11-24T22:15:39,530 INFO [RS:0;bd6592c37bb2:40087-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 94f3ccf9f8a638fa974e1f69627832ef/B of 94f3ccf9f8a638fa974e1f69627832ef into 3ef534ba6992421fb44a53ef9a12e571(size=12.4 K), total size for store is 12.4 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-24T22:15:39,530 DEBUG [RS:0;bd6592c37bb2:40087-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 94f3ccf9f8a638fa974e1f69627832ef: 2024-11-24T22:15:39,530 INFO [RS:0;bd6592c37bb2:40087-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732486521610.94f3ccf9f8a638fa974e1f69627832ef., storeName=94f3ccf9f8a638fa974e1f69627832ef/B, priority=13, startTime=1732486539106; duration=0sec 2024-11-24T22:15:39,530 DEBUG [RS:0;bd6592c37bb2:40087-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-24T22:15:39,530 DEBUG [RS:0;bd6592c37bb2:40087-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 94f3ccf9f8a638fa974e1f69627832ef:B 2024-11-24T22:15:39,531 DEBUG [RS:0;bd6592c37bb2:40087-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-24T22:15:39,531 DEBUG [RS:0;bd6592c37bb2:40087-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36863 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-24T22:15:39,531 DEBUG [RS:0;bd6592c37bb2:40087-longCompactions-0 {}] regionserver.HStore(1541): 94f3ccf9f8a638fa974e1f69627832ef/C is initiating minor compaction (all files) 2024-11-24T22:15:39,531 INFO [RS:0;bd6592c37bb2:40087-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 94f3ccf9f8a638fa974e1f69627832ef/C in TestAcidGuarantees,,1732486521610.94f3ccf9f8a638fa974e1f69627832ef. 2024-11-24T22:15:39,531 INFO [RS:0;bd6592c37bb2:40087-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/94f3ccf9f8a638fa974e1f69627832ef/C/d067a887518d4c33a3df8668caf4c2da, hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/94f3ccf9f8a638fa974e1f69627832ef/C/644e239d0c2a44909d1456f6cd2cb2b2, hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/94f3ccf9f8a638fa974e1f69627832ef/C/1f70e3bbb0e24ae8b788a7fc0abb2288] into tmpdir=hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/94f3ccf9f8a638fa974e1f69627832ef/.tmp, totalSize=36.0 K 2024-11-24T22:15:39,532 DEBUG [RS:0;bd6592c37bb2:40087-longCompactions-0 {}] compactions.Compactor(225): Compacting d067a887518d4c33a3df8668caf4c2da, keycount=150, bloomtype=ROW, size=12.3 K, encoding=NONE, compression=NONE, seqNum=175, earliestPutTs=1732486534720 2024-11-24T22:15:39,532 DEBUG [RS:0;bd6592c37bb2:40087-longCompactions-0 {}] compactions.Compactor(225): Compacting 644e239d0c2a44909d1456f6cd2cb2b2, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=189, earliestPutTs=1732486535727 2024-11-24T22:15:39,532 DEBUG [RS:0;bd6592c37bb2:40087-longCompactions-0 {}] compactions.Compactor(225): Compacting 1f70e3bbb0e24ae8b788a7fc0abb2288, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=211, earliestPutTs=1732486537048 2024-11-24T22:15:39,538 INFO [RS:0;bd6592c37bb2:40087-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 94f3ccf9f8a638fa974e1f69627832ef#C#compaction#350 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-24T22:15:39,538 DEBUG [RS:0;bd6592c37bb2:40087-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/94f3ccf9f8a638fa974e1f69627832ef/.tmp/C/33635f7b1a774316b05468b61e43d862 is 50, key is test_row_0/C:col10/1732486537050/Put/seqid=0 2024-11-24T22:15:39,541 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45929 is added to blk_1073742244_1420 (size=12663) 2024-11-24T22:15:39,607 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bd6592c37bb2:0-1 {event_type=RS_FLUSH_REGIONS, pid=104}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T22:15:39,611 INFO [RS_FLUSH_OPERATIONS-regionserver/bd6592c37bb2:0-1 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411245775428d17684f829ea4eb940d9c82dd_94f3ccf9f8a638fa974e1f69627832ef to hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411245775428d17684f829ea4eb940d9c82dd_94f3ccf9f8a638fa974e1f69627832ef 2024-11-24T22:15:39,611 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bd6592c37bb2:0-1 {event_type=RS_FLUSH_REGIONS, pid=104}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/94f3ccf9f8a638fa974e1f69627832ef/.tmp/A/93e608ac185c42b39eac1782462a5302, store: [table=TestAcidGuarantees family=A region=94f3ccf9f8a638fa974e1f69627832ef] 2024-11-24T22:15:39,612 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bd6592c37bb2:0-1 {event_type=RS_FLUSH_REGIONS, pid=104}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/94f3ccf9f8a638fa974e1f69627832ef/.tmp/A/93e608ac185c42b39eac1782462a5302 is 175, key is test_row_0/A:col10/1732486538260/Put/seqid=0 2024-11-24T22:15:39,615 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45929 is added to blk_1073742245_1421 (size=31105) 2024-11-24T22:15:39,630 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40087 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94f3ccf9f8a638fa974e1f69627832ef, server=bd6592c37bb2,40087,1732486423713 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-24T22:15:39,630 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40087 {}] ipc.CallRunner(138): callId: 66 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:32832 deadline: 1732486599620, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94f3ccf9f8a638fa974e1f69627832ef, server=bd6592c37bb2,40087,1732486423713 2024-11-24T22:15:39,630 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40087 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94f3ccf9f8a638fa974e1f69627832ef, server=bd6592c37bb2,40087,1732486423713 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-24T22:15:39,630 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732486521610.94f3ccf9f8a638fa974e1f69627832ef., hostname=bd6592c37bb2,40087,1732486423713, seqNum=5 , the old value is region=TestAcidGuarantees,,1732486521610.94f3ccf9f8a638fa974e1f69627832ef., hostname=bd6592c37bb2,40087,1732486423713, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94f3ccf9f8a638fa974e1f69627832ef, server=bd6592c37bb2,40087,1732486423713 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T22:15:39,630 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40087 {}] ipc.CallRunner(138): callId: 83 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:32774 deadline: 1732486599620, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94f3ccf9f8a638fa974e1f69627832ef, server=bd6592c37bb2,40087,1732486423713 2024-11-24T22:15:39,630 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732486521610.94f3ccf9f8a638fa974e1f69627832ef., hostname=bd6592c37bb2,40087,1732486423713, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94f3ccf9f8a638fa974e1f69627832ef, server=bd6592c37bb2,40087,1732486423713 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T22:15:39,630 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732486521610.94f3ccf9f8a638fa974e1f69627832ef., hostname=bd6592c37bb2,40087,1732486423713, seqNum=5 because the exception is null or not the one we care about 2024-11-24T22:15:39,630 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732486521610.94f3ccf9f8a638fa974e1f69627832ef., hostname=bd6592c37bb2,40087,1732486423713, seqNum=5 , the old value is region=TestAcidGuarantees,,1732486521610.94f3ccf9f8a638fa974e1f69627832ef., hostname=bd6592c37bb2,40087,1732486423713, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94f3ccf9f8a638fa974e1f69627832ef, server=bd6592c37bb2,40087,1732486423713 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T22:15:39,630 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732486521610.94f3ccf9f8a638fa974e1f69627832ef., hostname=bd6592c37bb2,40087,1732486423713, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94f3ccf9f8a638fa974e1f69627832ef, server=bd6592c37bb2,40087,1732486423713 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T22:15:39,631 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732486521610.94f3ccf9f8a638fa974e1f69627832ef., hostname=bd6592c37bb2,40087,1732486423713, seqNum=5 because the exception is null or not the one we care about 2024-11-24T22:15:39,638 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40087 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94f3ccf9f8a638fa974e1f69627832ef, server=bd6592c37bb2,40087,1732486423713 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-24T22:15:39,638 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40087 {}] ipc.CallRunner(138): callId: 84 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:32846 deadline: 1732486599631, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94f3ccf9f8a638fa974e1f69627832ef, server=bd6592c37bb2,40087,1732486423713 2024-11-24T22:15:39,638 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732486521610.94f3ccf9f8a638fa974e1f69627832ef., hostname=bd6592c37bb2,40087,1732486423713, seqNum=5 , the old value is region=TestAcidGuarantees,,1732486521610.94f3ccf9f8a638fa974e1f69627832ef., hostname=bd6592c37bb2,40087,1732486423713, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94f3ccf9f8a638fa974e1f69627832ef, server=bd6592c37bb2,40087,1732486423713 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T22:15:39,638 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732486521610.94f3ccf9f8a638fa974e1f69627832ef., hostname=bd6592c37bb2,40087,1732486423713, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94f3ccf9f8a638fa974e1f69627832ef, server=bd6592c37bb2,40087,1732486423713 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T22:15:39,638 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732486521610.94f3ccf9f8a638fa974e1f69627832ef., hostname=bd6592c37bb2,40087,1732486423713, seqNum=5 because the exception is null or not the one we care about 2024-11-24T22:15:39,851 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40087 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94f3ccf9f8a638fa974e1f69627832ef, server=bd6592c37bb2,40087,1732486423713 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-24T22:15:39,851 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40087 {}] ipc.CallRunner(138): callId: 67 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:32832 deadline: 1732486599840, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94f3ccf9f8a638fa974e1f69627832ef, server=bd6592c37bb2,40087,1732486423713 2024-11-24T22:15:39,851 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40087 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94f3ccf9f8a638fa974e1f69627832ef, server=bd6592c37bb2,40087,1732486423713 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-24T22:15:39,851 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40087 {}] ipc.CallRunner(138): callId: 84 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:32774 deadline: 1732486599840, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94f3ccf9f8a638fa974e1f69627832ef, server=bd6592c37bb2,40087,1732486423713 2024-11-24T22:15:39,852 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732486521610.94f3ccf9f8a638fa974e1f69627832ef., hostname=bd6592c37bb2,40087,1732486423713, seqNum=5 , the old value is region=TestAcidGuarantees,,1732486521610.94f3ccf9f8a638fa974e1f69627832ef., hostname=bd6592c37bb2,40087,1732486423713, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94f3ccf9f8a638fa974e1f69627832ef, server=bd6592c37bb2,40087,1732486423713 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T22:15:39,852 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732486521610.94f3ccf9f8a638fa974e1f69627832ef., hostname=bd6592c37bb2,40087,1732486423713, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94f3ccf9f8a638fa974e1f69627832ef, server=bd6592c37bb2,40087,1732486423713 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T22:15:39,852 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732486521610.94f3ccf9f8a638fa974e1f69627832ef., hostname=bd6592c37bb2,40087,1732486423713, seqNum=5 because the exception is null or not the one we care about 2024-11-24T22:15:39,852 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732486521610.94f3ccf9f8a638fa974e1f69627832ef., hostname=bd6592c37bb2,40087,1732486423713, seqNum=5 , the old value is region=TestAcidGuarantees,,1732486521610.94f3ccf9f8a638fa974e1f69627832ef., hostname=bd6592c37bb2,40087,1732486423713, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94f3ccf9f8a638fa974e1f69627832ef, server=bd6592c37bb2,40087,1732486423713 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T22:15:39,852 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732486521610.94f3ccf9f8a638fa974e1f69627832ef., hostname=bd6592c37bb2,40087,1732486423713, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94f3ccf9f8a638fa974e1f69627832ef, server=bd6592c37bb2,40087,1732486423713 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T22:15:39,852 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732486521610.94f3ccf9f8a638fa974e1f69627832ef., hostname=bd6592c37bb2,40087,1732486423713, seqNum=5 because the exception is null or not the one we care about 2024-11-24T22:15:39,860 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40087 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94f3ccf9f8a638fa974e1f69627832ef, server=bd6592c37bb2,40087,1732486423713 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-24T22:15:39,860 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40087 {}] ipc.CallRunner(138): callId: 85 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:32846 deadline: 1732486599851, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94f3ccf9f8a638fa974e1f69627832ef, server=bd6592c37bb2,40087,1732486423713 2024-11-24T22:15:39,861 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732486521610.94f3ccf9f8a638fa974e1f69627832ef., hostname=bd6592c37bb2,40087,1732486423713, seqNum=5 , the old value is region=TestAcidGuarantees,,1732486521610.94f3ccf9f8a638fa974e1f69627832ef., hostname=bd6592c37bb2,40087,1732486423713, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94f3ccf9f8a638fa974e1f69627832ef, server=bd6592c37bb2,40087,1732486423713 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T22:15:39,861 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732486521610.94f3ccf9f8a638fa974e1f69627832ef., hostname=bd6592c37bb2,40087,1732486423713, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94f3ccf9f8a638fa974e1f69627832ef, server=bd6592c37bb2,40087,1732486423713 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T22:15:39,861 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732486521610.94f3ccf9f8a638fa974e1f69627832ef., hostname=bd6592c37bb2,40087,1732486423713, seqNum=5 because the exception is null or not the one we care about 2024-11-24T22:15:39,932 DEBUG [RS:0;bd6592c37bb2:40087-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/94f3ccf9f8a638fa974e1f69627832ef/.tmp/A/090547ea69e9495fb9b887db92c6c3da as hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/94f3ccf9f8a638fa974e1f69627832ef/A/090547ea69e9495fb9b887db92c6c3da 2024-11-24T22:15:39,936 INFO [RS:0;bd6592c37bb2:40087-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 94f3ccf9f8a638fa974e1f69627832ef/A of 94f3ccf9f8a638fa974e1f69627832ef into 090547ea69e9495fb9b887db92c6c3da(size=30.9 K), total size for store is 30.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-24T22:15:39,936 DEBUG [RS:0;bd6592c37bb2:40087-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 94f3ccf9f8a638fa974e1f69627832ef: 2024-11-24T22:15:39,936 INFO [RS:0;bd6592c37bb2:40087-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732486521610.94f3ccf9f8a638fa974e1f69627832ef., storeName=94f3ccf9f8a638fa974e1f69627832ef/A, priority=13, startTime=1732486539106; duration=0sec 2024-11-24T22:15:39,936 DEBUG [RS:0;bd6592c37bb2:40087-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-24T22:15:39,936 DEBUG [RS:0;bd6592c37bb2:40087-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 94f3ccf9f8a638fa974e1f69627832ef:A 2024-11-24T22:15:39,945 DEBUG [RS:0;bd6592c37bb2:40087-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/94f3ccf9f8a638fa974e1f69627832ef/.tmp/C/33635f7b1a774316b05468b61e43d862 as hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/94f3ccf9f8a638fa974e1f69627832ef/C/33635f7b1a774316b05468b61e43d862 2024-11-24T22:15:39,949 INFO [RS:0;bd6592c37bb2:40087-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 94f3ccf9f8a638fa974e1f69627832ef/C of 94f3ccf9f8a638fa974e1f69627832ef into 33635f7b1a774316b05468b61e43d862(size=12.4 K), total size for store is 12.4 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-24T22:15:39,949 DEBUG [RS:0;bd6592c37bb2:40087-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 94f3ccf9f8a638fa974e1f69627832ef: 2024-11-24T22:15:39,949 INFO [RS:0;bd6592c37bb2:40087-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732486521610.94f3ccf9f8a638fa974e1f69627832ef., storeName=94f3ccf9f8a638fa974e1f69627832ef/C, priority=13, startTime=1732486539106; duration=0sec 2024-11-24T22:15:39,949 DEBUG [RS:0;bd6592c37bb2:40087-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-24T22:15:39,949 DEBUG [RS:0;bd6592c37bb2:40087-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 94f3ccf9f8a638fa974e1f69627832ef:C 2024-11-24T22:15:39,952 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40087 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94f3ccf9f8a638fa974e1f69627832ef, server=bd6592c37bb2,40087,1732486423713 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-24T22:15:39,952 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40087 {}] ipc.CallRunner(138): callId: 76 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:32822 deadline: 1732486599940, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94f3ccf9f8a638fa974e1f69627832ef, server=bd6592c37bb2,40087,1732486423713 2024-11-24T22:15:39,953 WARN [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRpcRetryingCaller(177): Call to bd6592c37bb2,40087,1732486423713 for 'test_row_2' in 94f3ccf9f8a638fa974e1f69627832ef of TestAcidGuarantees failed, tries = 7, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 4211 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94f3ccf9f8a638fa974e1f69627832ef, server=bd6592c37bb2,40087,1732486423713 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor44.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94f3ccf9f8a638fa974e1f69627832ef, server=bd6592c37bb2,40087,1732486423713 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-24T22:15:39,953 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732486521610.94f3ccf9f8a638fa974e1f69627832ef., hostname=bd6592c37bb2,40087,1732486423713, seqNum=5 , the old value is region=TestAcidGuarantees,,1732486521610.94f3ccf9f8a638fa974e1f69627832ef., hostname=bd6592c37bb2,40087,1732486423713, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94f3ccf9f8a638fa974e1f69627832ef, server=bd6592c37bb2,40087,1732486423713 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T22:15:39,953 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732486521610.94f3ccf9f8a638fa974e1f69627832ef., hostname=bd6592c37bb2,40087,1732486423713, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94f3ccf9f8a638fa974e1f69627832ef, server=bd6592c37bb2,40087,1732486423713 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T22:15:39,953 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732486521610.94f3ccf9f8a638fa974e1f69627832ef., hostname=bd6592c37bb2,40087,1732486423713, seqNum=5 because the exception is null or not the one we care about 2024-11-24T22:15:40,016 INFO [RS_FLUSH_OPERATIONS-regionserver/bd6592c37bb2:0-1 {event_type=RS_FLUSH_REGIONS, pid=104}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=224, memsize=22.4 K, hasBloomFilter=true, into tmp file hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/94f3ccf9f8a638fa974e1f69627832ef/.tmp/A/93e608ac185c42b39eac1782462a5302 2024-11-24T22:15:40,023 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bd6592c37bb2:0-1 {event_type=RS_FLUSH_REGIONS, pid=104}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/94f3ccf9f8a638fa974e1f69627832ef/.tmp/B/5cbd879d83374485b483f18f6194ad70 is 50, key is test_row_0/B:col10/1732486538260/Put/seqid=0 2024-11-24T22:15:40,026 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45929 is added to blk_1073742246_1422 (size=12151) 2024-11-24T22:15:40,167 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40087 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94f3ccf9f8a638fa974e1f69627832ef, server=bd6592c37bb2,40087,1732486423713 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-24T22:15:40,168 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40087 {}] ipc.CallRunner(138): callId: 68 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:32832 deadline: 1732486600161, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94f3ccf9f8a638fa974e1f69627832ef, server=bd6592c37bb2,40087,1732486423713 2024-11-24T22:15:40,168 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40087 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94f3ccf9f8a638fa974e1f69627832ef, server=bd6592c37bb2,40087,1732486423713 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-24T22:15:40,168 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732486521610.94f3ccf9f8a638fa974e1f69627832ef., hostname=bd6592c37bb2,40087,1732486423713, seqNum=5 , the old value is region=TestAcidGuarantees,,1732486521610.94f3ccf9f8a638fa974e1f69627832ef., hostname=bd6592c37bb2,40087,1732486423713, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94f3ccf9f8a638fa974e1f69627832ef, server=bd6592c37bb2,40087,1732486423713 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T22:15:40,168 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732486521610.94f3ccf9f8a638fa974e1f69627832ef., hostname=bd6592c37bb2,40087,1732486423713, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94f3ccf9f8a638fa974e1f69627832ef, server=bd6592c37bb2,40087,1732486423713 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T22:15:40,168 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40087 {}] ipc.CallRunner(138): callId: 85 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:32774 deadline: 1732486600161, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94f3ccf9f8a638fa974e1f69627832ef, server=bd6592c37bb2,40087,1732486423713 2024-11-24T22:15:40,168 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732486521610.94f3ccf9f8a638fa974e1f69627832ef., hostname=bd6592c37bb2,40087,1732486423713, seqNum=5 because the exception is null or not the one we care about 2024-11-24T22:15:40,168 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732486521610.94f3ccf9f8a638fa974e1f69627832ef., hostname=bd6592c37bb2,40087,1732486423713, seqNum=5 , the old value is region=TestAcidGuarantees,,1732486521610.94f3ccf9f8a638fa974e1f69627832ef., hostname=bd6592c37bb2,40087,1732486423713, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94f3ccf9f8a638fa974e1f69627832ef, server=bd6592c37bb2,40087,1732486423713 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T22:15:40,168 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732486521610.94f3ccf9f8a638fa974e1f69627832ef., hostname=bd6592c37bb2,40087,1732486423713, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94f3ccf9f8a638fa974e1f69627832ef, server=bd6592c37bb2,40087,1732486423713 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T22:15:40,168 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732486521610.94f3ccf9f8a638fa974e1f69627832ef., hostname=bd6592c37bb2,40087,1732486423713, seqNum=5 because the exception is null or not the one we care about 2024-11-24T22:15:40,182 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40087 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94f3ccf9f8a638fa974e1f69627832ef, server=bd6592c37bb2,40087,1732486423713 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-24T22:15:40,182 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40087 {}] ipc.CallRunner(138): callId: 86 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:32846 deadline: 1732486600170, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94f3ccf9f8a638fa974e1f69627832ef, server=bd6592c37bb2,40087,1732486423713 2024-11-24T22:15:40,183 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732486521610.94f3ccf9f8a638fa974e1f69627832ef., hostname=bd6592c37bb2,40087,1732486423713, seqNum=5 , the old value is region=TestAcidGuarantees,,1732486521610.94f3ccf9f8a638fa974e1f69627832ef., hostname=bd6592c37bb2,40087,1732486423713, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94f3ccf9f8a638fa974e1f69627832ef, server=bd6592c37bb2,40087,1732486423713 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T22:15:40,183 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732486521610.94f3ccf9f8a638fa974e1f69627832ef., hostname=bd6592c37bb2,40087,1732486423713, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94f3ccf9f8a638fa974e1f69627832ef, server=bd6592c37bb2,40087,1732486423713 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T22:15:40,183 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732486521610.94f3ccf9f8a638fa974e1f69627832ef., hostname=bd6592c37bb2,40087,1732486423713, seqNum=5 because the exception is null or not the one we care about 2024-11-24T22:15:40,426 INFO [RS_FLUSH_OPERATIONS-regionserver/bd6592c37bb2:0-1 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=224 (bloomFilter=true), to=hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/94f3ccf9f8a638fa974e1f69627832ef/.tmp/B/5cbd879d83374485b483f18f6194ad70 2024-11-24T22:15:40,433 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bd6592c37bb2:0-1 {event_type=RS_FLUSH_REGIONS, pid=104}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/94f3ccf9f8a638fa974e1f69627832ef/.tmp/C/25ca7dfcdf7c4069ba937e2d7f4fc7e7 is 50, key is test_row_0/C:col10/1732486538260/Put/seqid=0 2024-11-24T22:15:40,438 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45929 is added to blk_1073742247_1423 (size=12151) 2024-11-24T22:15:40,679 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40087 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94f3ccf9f8a638fa974e1f69627832ef, server=bd6592c37bb2,40087,1732486423713 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-24T22:15:40,679 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40087 {}] ipc.CallRunner(138): callId: 86 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:32774 deadline: 1732486600671, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94f3ccf9f8a638fa974e1f69627832ef, server=bd6592c37bb2,40087,1732486423713 2024-11-24T22:15:40,679 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732486521610.94f3ccf9f8a638fa974e1f69627832ef., hostname=bd6592c37bb2,40087,1732486423713, seqNum=5 , the old value is region=TestAcidGuarantees,,1732486521610.94f3ccf9f8a638fa974e1f69627832ef., hostname=bd6592c37bb2,40087,1732486423713, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94f3ccf9f8a638fa974e1f69627832ef, server=bd6592c37bb2,40087,1732486423713 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T22:15:40,679 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732486521610.94f3ccf9f8a638fa974e1f69627832ef., hostname=bd6592c37bb2,40087,1732486423713, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94f3ccf9f8a638fa974e1f69627832ef, server=bd6592c37bb2,40087,1732486423713 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T22:15:40,679 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732486521610.94f3ccf9f8a638fa974e1f69627832ef., hostname=bd6592c37bb2,40087,1732486423713, seqNum=5 because the exception is null or not the one we care about 2024-11-24T22:15:40,691 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40087 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94f3ccf9f8a638fa974e1f69627832ef, server=bd6592c37bb2,40087,1732486423713 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-24T22:15:40,691 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40087 {}] ipc.CallRunner(138): callId: 69 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:32832 deadline: 1732486600681, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94f3ccf9f8a638fa974e1f69627832ef, server=bd6592c37bb2,40087,1732486423713 2024-11-24T22:15:40,691 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732486521610.94f3ccf9f8a638fa974e1f69627832ef., hostname=bd6592c37bb2,40087,1732486423713, seqNum=5 , the old value is region=TestAcidGuarantees,,1732486521610.94f3ccf9f8a638fa974e1f69627832ef., hostname=bd6592c37bb2,40087,1732486423713, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94f3ccf9f8a638fa974e1f69627832ef, server=bd6592c37bb2,40087,1732486423713 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T22:15:40,691 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732486521610.94f3ccf9f8a638fa974e1f69627832ef., hostname=bd6592c37bb2,40087,1732486423713, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94f3ccf9f8a638fa974e1f69627832ef, server=bd6592c37bb2,40087,1732486423713 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T22:15:40,691 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732486521610.94f3ccf9f8a638fa974e1f69627832ef., hostname=bd6592c37bb2,40087,1732486423713, seqNum=5 because the exception is null or not the one we care about 2024-11-24T22:15:40,695 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40087 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94f3ccf9f8a638fa974e1f69627832ef, server=bd6592c37bb2,40087,1732486423713 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-24T22:15:40,695 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40087 {}] ipc.CallRunner(138): callId: 87 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:32846 deadline: 1732486600690, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94f3ccf9f8a638fa974e1f69627832ef, server=bd6592c37bb2,40087,1732486423713 2024-11-24T22:15:40,695 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732486521610.94f3ccf9f8a638fa974e1f69627832ef., hostname=bd6592c37bb2,40087,1732486423713, seqNum=5 , the old value is region=TestAcidGuarantees,,1732486521610.94f3ccf9f8a638fa974e1f69627832ef., hostname=bd6592c37bb2,40087,1732486423713, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94f3ccf9f8a638fa974e1f69627832ef, server=bd6592c37bb2,40087,1732486423713 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T22:15:40,695 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732486521610.94f3ccf9f8a638fa974e1f69627832ef., hostname=bd6592c37bb2,40087,1732486423713, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94f3ccf9f8a638fa974e1f69627832ef, server=bd6592c37bb2,40087,1732486423713 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T22:15:40,695 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732486521610.94f3ccf9f8a638fa974e1f69627832ef., hostname=bd6592c37bb2,40087,1732486423713, seqNum=5 because the exception is null or not the one we care about 2024-11-24T22:15:40,838 INFO [RS_FLUSH_OPERATIONS-regionserver/bd6592c37bb2:0-1 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=224 (bloomFilter=true), to=hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/94f3ccf9f8a638fa974e1f69627832ef/.tmp/C/25ca7dfcdf7c4069ba937e2d7f4fc7e7 2024-11-24T22:15:40,843 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bd6592c37bb2:0-1 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/94f3ccf9f8a638fa974e1f69627832ef/.tmp/A/93e608ac185c42b39eac1782462a5302 as hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/94f3ccf9f8a638fa974e1f69627832ef/A/93e608ac185c42b39eac1782462a5302 2024-11-24T22:15:40,846 INFO [RS_FLUSH_OPERATIONS-regionserver/bd6592c37bb2:0-1 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/94f3ccf9f8a638fa974e1f69627832ef/A/93e608ac185c42b39eac1782462a5302, entries=150, sequenceid=224, filesize=30.4 K 2024-11-24T22:15:40,847 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bd6592c37bb2:0-1 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/94f3ccf9f8a638fa974e1f69627832ef/.tmp/B/5cbd879d83374485b483f18f6194ad70 as hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/94f3ccf9f8a638fa974e1f69627832ef/B/5cbd879d83374485b483f18f6194ad70 2024-11-24T22:15:40,851 INFO [RS_FLUSH_OPERATIONS-regionserver/bd6592c37bb2:0-1 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/94f3ccf9f8a638fa974e1f69627832ef/B/5cbd879d83374485b483f18f6194ad70, entries=150, sequenceid=224, filesize=11.9 K 2024-11-24T22:15:40,851 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bd6592c37bb2:0-1 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/94f3ccf9f8a638fa974e1f69627832ef/.tmp/C/25ca7dfcdf7c4069ba937e2d7f4fc7e7 as hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/94f3ccf9f8a638fa974e1f69627832ef/C/25ca7dfcdf7c4069ba937e2d7f4fc7e7 2024-11-24T22:15:40,855 INFO [RS_FLUSH_OPERATIONS-regionserver/bd6592c37bb2:0-1 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/94f3ccf9f8a638fa974e1f69627832ef/C/25ca7dfcdf7c4069ba937e2d7f4fc7e7, entries=150, sequenceid=224, filesize=11.9 K 2024-11-24T22:15:40,856 INFO [RS_FLUSH_OPERATIONS-regionserver/bd6592c37bb2:0-1 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.HRegion(3140): Finished flush of dataSize ~67.09 KB/68700, heapSize ~199.92 KB/204720, currentSize=107.34 KB/109920 for 94f3ccf9f8a638fa974e1f69627832ef in 1661ms, sequenceid=224, compaction requested=false 2024-11-24T22:15:40,856 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bd6592c37bb2:0-1 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.HRegion(2603): Flush status journal for 94f3ccf9f8a638fa974e1f69627832ef: 2024-11-24T22:15:40,856 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bd6592c37bb2:0-1 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732486521610.94f3ccf9f8a638fa974e1f69627832ef. 2024-11-24T22:15:40,856 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bd6592c37bb2:0-1 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=104 2024-11-24T22:15:40,856 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37701 {}] master.HMaster(4169): Remote procedure done, pid=104 2024-11-24T22:15:40,859 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=104, resume processing ppid=103 2024-11-24T22:15:40,859 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=104, ppid=103, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 3.6530 sec 2024-11-24T22:15:40,861 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=103, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=103, table=TestAcidGuarantees in 3.6570 sec 2024-11-24T22:15:41,361 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37701 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=103 2024-11-24T22:15:41,361 INFO [RPCClient-NioEventLoopGroup-4-8 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-24T22:15:41,362 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37701 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-24T22:15:41,363 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37701 {}] procedure2.ProcedureExecutor(1139): Stored pid=105, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=105, table=TestAcidGuarantees 2024-11-24T22:15:41,364 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37701 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=105 2024-11-24T22:15:41,364 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=105, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=105, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-24T22:15:41,364 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=105, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=105, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-24T22:15:41,365 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=106, ppid=105, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-24T22:15:41,471 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37701 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=105 2024-11-24T22:15:41,517 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=40087 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=106 2024-11-24T22:15:41,517 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bd6592c37bb2:0-2 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732486521610.94f3ccf9f8a638fa974e1f69627832ef. 2024-11-24T22:15:41,517 INFO [RS_FLUSH_OPERATIONS-regionserver/bd6592c37bb2:0-2 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.HRegion(2902): Flushing 94f3ccf9f8a638fa974e1f69627832ef 3/3 column families, dataSize=107.34 KB heapSize=319.50 KB 2024-11-24T22:15:41,518 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bd6592c37bb2:0-2 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 94f3ccf9f8a638fa974e1f69627832ef, store=A 2024-11-24T22:15:41,518 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bd6592c37bb2:0-2 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-24T22:15:41,518 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bd6592c37bb2:0-2 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 94f3ccf9f8a638fa974e1f69627832ef, store=B 2024-11-24T22:15:41,518 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bd6592c37bb2:0-2 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-24T22:15:41,518 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bd6592c37bb2:0-2 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 94f3ccf9f8a638fa974e1f69627832ef, store=C 2024-11-24T22:15:41,518 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bd6592c37bb2:0-2 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-24T22:15:41,525 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bd6592c37bb2:0-2 {event_type=RS_FLUSH_REGIONS, pid=106}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241124067acc14c06a447ea8f164340f4cdd56_94f3ccf9f8a638fa974e1f69627832ef is 50, key is test_row_0/A:col10/1732486539479/Put/seqid=0 2024-11-24T22:15:41,528 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45929 is added to blk_1073742248_1424 (size=12304) 2024-11-24T22:15:41,680 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37701 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=105 2024-11-24T22:15:41,696 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40087 {}] regionserver.HRegion(8855): Flush requested on 94f3ccf9f8a638fa974e1f69627832ef 2024-11-24T22:15:41,696 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732486521610.94f3ccf9f8a638fa974e1f69627832ef. as already flushing 2024-11-24T22:15:41,728 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40087 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94f3ccf9f8a638fa974e1f69627832ef, server=bd6592c37bb2,40087,1732486423713 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-24T22:15:41,728 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40087 {}] ipc.CallRunner(138): callId: 91 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:32774 deadline: 1732486601725, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94f3ccf9f8a638fa974e1f69627832ef, server=bd6592c37bb2,40087,1732486423713 2024-11-24T22:15:41,728 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732486521610.94f3ccf9f8a638fa974e1f69627832ef., hostname=bd6592c37bb2,40087,1732486423713, seqNum=5 , the old value is region=TestAcidGuarantees,,1732486521610.94f3ccf9f8a638fa974e1f69627832ef., hostname=bd6592c37bb2,40087,1732486423713, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94f3ccf9f8a638fa974e1f69627832ef, server=bd6592c37bb2,40087,1732486423713 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T22:15:41,728 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732486521610.94f3ccf9f8a638fa974e1f69627832ef., hostname=bd6592c37bb2,40087,1732486423713, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94f3ccf9f8a638fa974e1f69627832ef, server=bd6592c37bb2,40087,1732486423713 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T22:15:41,728 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732486521610.94f3ccf9f8a638fa974e1f69627832ef., hostname=bd6592c37bb2,40087,1732486423713, seqNum=5 because the exception is null or not the one we care about 2024-11-24T22:15:41,736 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40087 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94f3ccf9f8a638fa974e1f69627832ef, server=bd6592c37bb2,40087,1732486423713 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-24T22:15:41,737 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40087 {}] ipc.CallRunner(138): callId: 73 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:32832 deadline: 1732486601727, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94f3ccf9f8a638fa974e1f69627832ef, server=bd6592c37bb2,40087,1732486423713 2024-11-24T22:15:41,737 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732486521610.94f3ccf9f8a638fa974e1f69627832ef., hostname=bd6592c37bb2,40087,1732486423713, seqNum=5 , the old value is region=TestAcidGuarantees,,1732486521610.94f3ccf9f8a638fa974e1f69627832ef., hostname=bd6592c37bb2,40087,1732486423713, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94f3ccf9f8a638fa974e1f69627832ef, server=bd6592c37bb2,40087,1732486423713 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T22:15:41,737 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732486521610.94f3ccf9f8a638fa974e1f69627832ef., hostname=bd6592c37bb2,40087,1732486423713, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94f3ccf9f8a638fa974e1f69627832ef, server=bd6592c37bb2,40087,1732486423713 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T22:15:41,737 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732486521610.94f3ccf9f8a638fa974e1f69627832ef., hostname=bd6592c37bb2,40087,1732486423713, seqNum=5 because the exception is null or not the one we care about 2024-11-24T22:15:41,741 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40087 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94f3ccf9f8a638fa974e1f69627832ef, server=bd6592c37bb2,40087,1732486423713 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-24T22:15:41,741 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40087 {}] ipc.CallRunner(138): callId: 91 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:32846 deadline: 1732486601728, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94f3ccf9f8a638fa974e1f69627832ef, server=bd6592c37bb2,40087,1732486423713 2024-11-24T22:15:41,741 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732486521610.94f3ccf9f8a638fa974e1f69627832ef., hostname=bd6592c37bb2,40087,1732486423713, seqNum=5 , the old value is region=TestAcidGuarantees,,1732486521610.94f3ccf9f8a638fa974e1f69627832ef., hostname=bd6592c37bb2,40087,1732486423713, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94f3ccf9f8a638fa974e1f69627832ef, server=bd6592c37bb2,40087,1732486423713 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T22:15:41,741 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732486521610.94f3ccf9f8a638fa974e1f69627832ef., hostname=bd6592c37bb2,40087,1732486423713, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94f3ccf9f8a638fa974e1f69627832ef, server=bd6592c37bb2,40087,1732486423713 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T22:15:41,741 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732486521610.94f3ccf9f8a638fa974e1f69627832ef., hostname=bd6592c37bb2,40087,1732486423713, seqNum=5 because the exception is null or not the one we care about 2024-11-24T22:15:41,841 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40087 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94f3ccf9f8a638fa974e1f69627832ef, server=bd6592c37bb2,40087,1732486423713 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-24T22:15:41,841 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40087 {}] ipc.CallRunner(138): callId: 92 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:32774 deadline: 1732486601831, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94f3ccf9f8a638fa974e1f69627832ef, server=bd6592c37bb2,40087,1732486423713 2024-11-24T22:15:41,841 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732486521610.94f3ccf9f8a638fa974e1f69627832ef., hostname=bd6592c37bb2,40087,1732486423713, seqNum=5 , the old value is region=TestAcidGuarantees,,1732486521610.94f3ccf9f8a638fa974e1f69627832ef., hostname=bd6592c37bb2,40087,1732486423713, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94f3ccf9f8a638fa974e1f69627832ef, server=bd6592c37bb2,40087,1732486423713 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T22:15:41,841 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732486521610.94f3ccf9f8a638fa974e1f69627832ef., hostname=bd6592c37bb2,40087,1732486423713, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94f3ccf9f8a638fa974e1f69627832ef, server=bd6592c37bb2,40087,1732486423713 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T22:15:41,841 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732486521610.94f3ccf9f8a638fa974e1f69627832ef., hostname=bd6592c37bb2,40087,1732486423713, seqNum=5 because the exception is null or not the one we care about 2024-11-24T22:15:41,851 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40087 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94f3ccf9f8a638fa974e1f69627832ef, server=bd6592c37bb2,40087,1732486423713 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-24T22:15:41,851 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40087 {}] ipc.CallRunner(138): callId: 74 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:32832 deadline: 1732486601840, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94f3ccf9f8a638fa974e1f69627832ef, server=bd6592c37bb2,40087,1732486423713 2024-11-24T22:15:41,852 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732486521610.94f3ccf9f8a638fa974e1f69627832ef., hostname=bd6592c37bb2,40087,1732486423713, seqNum=5 , the old value is region=TestAcidGuarantees,,1732486521610.94f3ccf9f8a638fa974e1f69627832ef., hostname=bd6592c37bb2,40087,1732486423713, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94f3ccf9f8a638fa974e1f69627832ef, server=bd6592c37bb2,40087,1732486423713 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T22:15:41,852 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732486521610.94f3ccf9f8a638fa974e1f69627832ef., hostname=bd6592c37bb2,40087,1732486423713, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94f3ccf9f8a638fa974e1f69627832ef, server=bd6592c37bb2,40087,1732486423713 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T22:15:41,852 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732486521610.94f3ccf9f8a638fa974e1f69627832ef., hostname=bd6592c37bb2,40087,1732486423713, seqNum=5 because the exception is null or not the one we care about 2024-11-24T22:15:41,859 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40087 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94f3ccf9f8a638fa974e1f69627832ef, server=bd6592c37bb2,40087,1732486423713 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-24T22:15:41,859 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40087 {}] ipc.CallRunner(138): callId: 92 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:32846 deadline: 1732486601850, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94f3ccf9f8a638fa974e1f69627832ef, server=bd6592c37bb2,40087,1732486423713 2024-11-24T22:15:41,860 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732486521610.94f3ccf9f8a638fa974e1f69627832ef., hostname=bd6592c37bb2,40087,1732486423713, seqNum=5 , the old value is region=TestAcidGuarantees,,1732486521610.94f3ccf9f8a638fa974e1f69627832ef., hostname=bd6592c37bb2,40087,1732486423713, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94f3ccf9f8a638fa974e1f69627832ef, server=bd6592c37bb2,40087,1732486423713 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T22:15:41,860 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732486521610.94f3ccf9f8a638fa974e1f69627832ef., hostname=bd6592c37bb2,40087,1732486423713, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94f3ccf9f8a638fa974e1f69627832ef, server=bd6592c37bb2,40087,1732486423713 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T22:15:41,860 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732486521610.94f3ccf9f8a638fa974e1f69627832ef., hostname=bd6592c37bb2,40087,1732486423713, seqNum=5 because the exception is null or not the one we care about 2024-11-24T22:15:41,929 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bd6592c37bb2:0-2 {event_type=RS_FLUSH_REGIONS, pid=106}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T22:15:41,932 INFO [RS_FLUSH_OPERATIONS-regionserver/bd6592c37bb2:0-2 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241124067acc14c06a447ea8f164340f4cdd56_94f3ccf9f8a638fa974e1f69627832ef to hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241124067acc14c06a447ea8f164340f4cdd56_94f3ccf9f8a638fa974e1f69627832ef 2024-11-24T22:15:41,933 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bd6592c37bb2:0-2 {event_type=RS_FLUSH_REGIONS, pid=106}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/94f3ccf9f8a638fa974e1f69627832ef/.tmp/A/57308f1fd93a41fa954bec339b0cb607, store: [table=TestAcidGuarantees family=A region=94f3ccf9f8a638fa974e1f69627832ef] 2024-11-24T22:15:41,934 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bd6592c37bb2:0-2 {event_type=RS_FLUSH_REGIONS, pid=106}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/94f3ccf9f8a638fa974e1f69627832ef/.tmp/A/57308f1fd93a41fa954bec339b0cb607 is 175, key is test_row_0/A:col10/1732486539479/Put/seqid=0 2024-11-24T22:15:41,937 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45929 is added to blk_1073742249_1425 (size=31105) 2024-11-24T22:15:41,990 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37701 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=105 2024-11-24T22:15:42,015 DEBUG [FsDatasetAsyncDiskServiceFixer {}] hbase.HBaseTestingUtil$FsDatasetAsyncDiskServiceFixer(597): NoSuchFieldException: threadGroup; It might because your Hadoop version > 3.2.3 or 3.3.4, See HBASE-27595 for details. 2024-11-24T22:15:42,060 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40087 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94f3ccf9f8a638fa974e1f69627832ef, server=bd6592c37bb2,40087,1732486423713 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-24T22:15:42,060 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40087 {}] ipc.CallRunner(138): callId: 93 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:32774 deadline: 1732486602051, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94f3ccf9f8a638fa974e1f69627832ef, server=bd6592c37bb2,40087,1732486423713 2024-11-24T22:15:42,060 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732486521610.94f3ccf9f8a638fa974e1f69627832ef., hostname=bd6592c37bb2,40087,1732486423713, seqNum=5 , the old value is region=TestAcidGuarantees,,1732486521610.94f3ccf9f8a638fa974e1f69627832ef., hostname=bd6592c37bb2,40087,1732486423713, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94f3ccf9f8a638fa974e1f69627832ef, server=bd6592c37bb2,40087,1732486423713 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T22:15:42,061 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732486521610.94f3ccf9f8a638fa974e1f69627832ef., hostname=bd6592c37bb2,40087,1732486423713, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94f3ccf9f8a638fa974e1f69627832ef, server=bd6592c37bb2,40087,1732486423713 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T22:15:42,061 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732486521610.94f3ccf9f8a638fa974e1f69627832ef., hostname=bd6592c37bb2,40087,1732486423713, seqNum=5 because the exception is null or not the one we care about 2024-11-24T22:15:42,066 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40087 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94f3ccf9f8a638fa974e1f69627832ef, server=bd6592c37bb2,40087,1732486423713 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-24T22:15:42,066 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40087 {}] ipc.CallRunner(138): callId: 75 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:32832 deadline: 1732486602060, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94f3ccf9f8a638fa974e1f69627832ef, server=bd6592c37bb2,40087,1732486423713 2024-11-24T22:15:42,066 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732486521610.94f3ccf9f8a638fa974e1f69627832ef., hostname=bd6592c37bb2,40087,1732486423713, seqNum=5 , the old value is region=TestAcidGuarantees,,1732486521610.94f3ccf9f8a638fa974e1f69627832ef., hostname=bd6592c37bb2,40087,1732486423713, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94f3ccf9f8a638fa974e1f69627832ef, server=bd6592c37bb2,40087,1732486423713 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T22:15:42,066 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732486521610.94f3ccf9f8a638fa974e1f69627832ef., hostname=bd6592c37bb2,40087,1732486423713, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94f3ccf9f8a638fa974e1f69627832ef, server=bd6592c37bb2,40087,1732486423713 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T22:15:42,067 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732486521610.94f3ccf9f8a638fa974e1f69627832ef., hostname=bd6592c37bb2,40087,1732486423713, seqNum=5 because the exception is null or not the one we care about 2024-11-24T22:15:42,079 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40087 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94f3ccf9f8a638fa974e1f69627832ef, server=bd6592c37bb2,40087,1732486423713 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-24T22:15:42,079 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40087 {}] ipc.CallRunner(138): callId: 93 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:32846 deadline: 1732486602070, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94f3ccf9f8a638fa974e1f69627832ef, server=bd6592c37bb2,40087,1732486423713 2024-11-24T22:15:42,080 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732486521610.94f3ccf9f8a638fa974e1f69627832ef., hostname=bd6592c37bb2,40087,1732486423713, seqNum=5 , the old value is region=TestAcidGuarantees,,1732486521610.94f3ccf9f8a638fa974e1f69627832ef., hostname=bd6592c37bb2,40087,1732486423713, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94f3ccf9f8a638fa974e1f69627832ef, server=bd6592c37bb2,40087,1732486423713 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T22:15:42,080 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732486521610.94f3ccf9f8a638fa974e1f69627832ef., hostname=bd6592c37bb2,40087,1732486423713, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94f3ccf9f8a638fa974e1f69627832ef, server=bd6592c37bb2,40087,1732486423713 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T22:15:42,080 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732486521610.94f3ccf9f8a638fa974e1f69627832ef., hostname=bd6592c37bb2,40087,1732486423713, seqNum=5 because the exception is null or not the one we care about 2024-11-24T22:15:42,339 INFO [RS_FLUSH_OPERATIONS-regionserver/bd6592c37bb2:0-2 {event_type=RS_FLUSH_REGIONS, pid=106}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=246, memsize=35.8 K, hasBloomFilter=true, into tmp file hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/94f3ccf9f8a638fa974e1f69627832ef/.tmp/A/57308f1fd93a41fa954bec339b0cb607 2024-11-24T22:15:42,351 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bd6592c37bb2:0-2 {event_type=RS_FLUSH_REGIONS, pid=106}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/94f3ccf9f8a638fa974e1f69627832ef/.tmp/B/149583da245f45179d6de44960e3e48e is 50, key is test_row_0/B:col10/1732486539479/Put/seqid=0 2024-11-24T22:15:42,379 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40087 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94f3ccf9f8a638fa974e1f69627832ef, server=bd6592c37bb2,40087,1732486423713 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-24T22:15:42,379 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40087 {}] ipc.CallRunner(138): callId: 94 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:32774 deadline: 1732486602371, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94f3ccf9f8a638fa974e1f69627832ef, server=bd6592c37bb2,40087,1732486423713 2024-11-24T22:15:42,379 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40087 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94f3ccf9f8a638fa974e1f69627832ef, server=bd6592c37bb2,40087,1732486423713 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-24T22:15:42,379 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40087 {}] ipc.CallRunner(138): callId: 76 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:32832 deadline: 1732486602371, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94f3ccf9f8a638fa974e1f69627832ef, server=bd6592c37bb2,40087,1732486423713 2024-11-24T22:15:42,380 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732486521610.94f3ccf9f8a638fa974e1f69627832ef., hostname=bd6592c37bb2,40087,1732486423713, seqNum=5 , the old value is region=TestAcidGuarantees,,1732486521610.94f3ccf9f8a638fa974e1f69627832ef., hostname=bd6592c37bb2,40087,1732486423713, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94f3ccf9f8a638fa974e1f69627832ef, server=bd6592c37bb2,40087,1732486423713 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T22:15:42,380 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732486521610.94f3ccf9f8a638fa974e1f69627832ef., hostname=bd6592c37bb2,40087,1732486423713, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94f3ccf9f8a638fa974e1f69627832ef, server=bd6592c37bb2,40087,1732486423713 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T22:15:42,380 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732486521610.94f3ccf9f8a638fa974e1f69627832ef., hostname=bd6592c37bb2,40087,1732486423713, seqNum=5 because the exception is null or not the one we care about 2024-11-24T22:15:42,381 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732486521610.94f3ccf9f8a638fa974e1f69627832ef., hostname=bd6592c37bb2,40087,1732486423713, seqNum=5 , the old value is region=TestAcidGuarantees,,1732486521610.94f3ccf9f8a638fa974e1f69627832ef., hostname=bd6592c37bb2,40087,1732486423713, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94f3ccf9f8a638fa974e1f69627832ef, server=bd6592c37bb2,40087,1732486423713 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T22:15:42,381 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732486521610.94f3ccf9f8a638fa974e1f69627832ef., hostname=bd6592c37bb2,40087,1732486423713, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94f3ccf9f8a638fa974e1f69627832ef, server=bd6592c37bb2,40087,1732486423713 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T22:15:42,381 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732486521610.94f3ccf9f8a638fa974e1f69627832ef., hostname=bd6592c37bb2,40087,1732486423713, seqNum=5 because the exception is null or not the one we care about 2024-11-24T22:15:42,386 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45929 is added to blk_1073742250_1426 (size=12151) 2024-11-24T22:15:42,391 INFO [RS_FLUSH_OPERATIONS-regionserver/bd6592c37bb2:0-2 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=35.78 KB at sequenceid=246 (bloomFilter=true), to=hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/94f3ccf9f8a638fa974e1f69627832ef/.tmp/B/149583da245f45179d6de44960e3e48e 2024-11-24T22:15:42,399 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40087 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94f3ccf9f8a638fa974e1f69627832ef, server=bd6592c37bb2,40087,1732486423713 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-24T22:15:42,399 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40087 {}] ipc.CallRunner(138): callId: 94 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:32846 deadline: 1732486602390, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94f3ccf9f8a638fa974e1f69627832ef, server=bd6592c37bb2,40087,1732486423713 2024-11-24T22:15:42,399 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732486521610.94f3ccf9f8a638fa974e1f69627832ef., hostname=bd6592c37bb2,40087,1732486423713, seqNum=5 , the old value is region=TestAcidGuarantees,,1732486521610.94f3ccf9f8a638fa974e1f69627832ef., hostname=bd6592c37bb2,40087,1732486423713, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94f3ccf9f8a638fa974e1f69627832ef, server=bd6592c37bb2,40087,1732486423713 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T22:15:42,399 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732486521610.94f3ccf9f8a638fa974e1f69627832ef., hostname=bd6592c37bb2,40087,1732486423713, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94f3ccf9f8a638fa974e1f69627832ef, server=bd6592c37bb2,40087,1732486423713 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T22:15:42,399 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732486521610.94f3ccf9f8a638fa974e1f69627832ef., hostname=bd6592c37bb2,40087,1732486423713, seqNum=5 because the exception is null or not the one we care about 2024-11-24T22:15:42,402 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bd6592c37bb2:0-2 {event_type=RS_FLUSH_REGIONS, pid=106}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/94f3ccf9f8a638fa974e1f69627832ef/.tmp/C/62d1d497a09e4e0fb4316beebe854732 is 50, key is test_row_0/C:col10/1732486539479/Put/seqid=0 2024-11-24T22:15:42,418 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45929 is added to blk_1073742251_1427 (size=12151) 2024-11-24T22:15:42,418 INFO [RS_FLUSH_OPERATIONS-regionserver/bd6592c37bb2:0-2 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=35.78 KB at sequenceid=246 (bloomFilter=true), to=hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/94f3ccf9f8a638fa974e1f69627832ef/.tmp/C/62d1d497a09e4e0fb4316beebe854732 2024-11-24T22:15:42,433 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bd6592c37bb2:0-2 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/94f3ccf9f8a638fa974e1f69627832ef/.tmp/A/57308f1fd93a41fa954bec339b0cb607 as hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/94f3ccf9f8a638fa974e1f69627832ef/A/57308f1fd93a41fa954bec339b0cb607 2024-11-24T22:15:42,436 INFO [RS_FLUSH_OPERATIONS-regionserver/bd6592c37bb2:0-2 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/94f3ccf9f8a638fa974e1f69627832ef/A/57308f1fd93a41fa954bec339b0cb607, entries=150, sequenceid=246, filesize=30.4 K 2024-11-24T22:15:42,439 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bd6592c37bb2:0-2 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/94f3ccf9f8a638fa974e1f69627832ef/.tmp/B/149583da245f45179d6de44960e3e48e as hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/94f3ccf9f8a638fa974e1f69627832ef/B/149583da245f45179d6de44960e3e48e 2024-11-24T22:15:42,442 INFO [RS_FLUSH_OPERATIONS-regionserver/bd6592c37bb2:0-2 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/94f3ccf9f8a638fa974e1f69627832ef/B/149583da245f45179d6de44960e3e48e, entries=150, sequenceid=246, filesize=11.9 K 2024-11-24T22:15:42,443 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bd6592c37bb2:0-2 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/94f3ccf9f8a638fa974e1f69627832ef/.tmp/C/62d1d497a09e4e0fb4316beebe854732 as hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/94f3ccf9f8a638fa974e1f69627832ef/C/62d1d497a09e4e0fb4316beebe854732 2024-11-24T22:15:42,446 INFO [RS_FLUSH_OPERATIONS-regionserver/bd6592c37bb2:0-2 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/94f3ccf9f8a638fa974e1f69627832ef/C/62d1d497a09e4e0fb4316beebe854732, entries=150, sequenceid=246, filesize=11.9 K 2024-11-24T22:15:42,447 INFO [RS_FLUSH_OPERATIONS-regionserver/bd6592c37bb2:0-2 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.HRegion(3140): Finished flush of dataSize ~107.34 KB/109920, heapSize ~319.45 KB/327120, currentSize=67.09 KB/68700 for 94f3ccf9f8a638fa974e1f69627832ef in 930ms, sequenceid=246, compaction requested=true 2024-11-24T22:15:42,447 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bd6592c37bb2:0-2 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.HRegion(2603): Flush status journal for 94f3ccf9f8a638fa974e1f69627832ef: 2024-11-24T22:15:42,447 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bd6592c37bb2:0-2 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732486521610.94f3ccf9f8a638fa974e1f69627832ef. 2024-11-24T22:15:42,447 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bd6592c37bb2:0-2 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=106 2024-11-24T22:15:42,447 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37701 {}] master.HMaster(4169): Remote procedure done, pid=106 2024-11-24T22:15:42,449 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=106, resume processing ppid=105 2024-11-24T22:15:42,449 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=106, ppid=105, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.0830 sec 2024-11-24T22:15:42,451 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=105, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=105, table=TestAcidGuarantees in 1.0880 sec 2024-11-24T22:15:42,501 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37701 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=105 2024-11-24T22:15:42,502 INFO [RPCClient-NioEventLoopGroup-4-8 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-24T22:15:42,502 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37701 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-24T22:15:42,503 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37701 {}] procedure2.ProcedureExecutor(1139): Stored pid=107, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=107, table=TestAcidGuarantees 2024-11-24T22:15:42,504 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37701 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=107 2024-11-24T22:15:42,504 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=107, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=107, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-24T22:15:42,505 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=107, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=107, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-24T22:15:42,505 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=108, ppid=107, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-24T22:15:42,611 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37701 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=107 2024-11-24T22:15:42,657 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=40087 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=108 2024-11-24T22:15:42,657 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bd6592c37bb2:0-0 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732486521610.94f3ccf9f8a638fa974e1f69627832ef. 2024-11-24T22:15:42,657 INFO [RS_FLUSH_OPERATIONS-regionserver/bd6592c37bb2:0-0 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.HRegion(2902): Flushing 94f3ccf9f8a638fa974e1f69627832ef 3/3 column families, dataSize=67.09 KB heapSize=199.97 KB 2024-11-24T22:15:42,657 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bd6592c37bb2:0-0 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 94f3ccf9f8a638fa974e1f69627832ef, store=A 2024-11-24T22:15:42,657 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bd6592c37bb2:0-0 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-24T22:15:42,658 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bd6592c37bb2:0-0 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 94f3ccf9f8a638fa974e1f69627832ef, store=B 2024-11-24T22:15:42,658 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bd6592c37bb2:0-0 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-24T22:15:42,658 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bd6592c37bb2:0-0 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 94f3ccf9f8a638fa974e1f69627832ef, store=C 2024-11-24T22:15:42,658 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bd6592c37bb2:0-0 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-24T22:15:42,664 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bd6592c37bb2:0-0 {event_type=RS_FLUSH_REGIONS, pid=108}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411247f8a7e400f40437488012456cbc688bf_94f3ccf9f8a638fa974e1f69627832ef is 50, key is test_row_0/A:col10/1732486541712/Put/seqid=0 2024-11-24T22:15:42,667 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45929 is added to blk_1073742252_1428 (size=12354) 2024-11-24T22:15:42,668 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bd6592c37bb2:0-0 {event_type=RS_FLUSH_REGIONS, pid=108}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T22:15:42,671 INFO [RS_FLUSH_OPERATIONS-regionserver/bd6592c37bb2:0-0 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411247f8a7e400f40437488012456cbc688bf_94f3ccf9f8a638fa974e1f69627832ef to hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411247f8a7e400f40437488012456cbc688bf_94f3ccf9f8a638fa974e1f69627832ef 2024-11-24T22:15:42,672 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bd6592c37bb2:0-0 {event_type=RS_FLUSH_REGIONS, pid=108}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/94f3ccf9f8a638fa974e1f69627832ef/.tmp/A/8d09d43826484065b9293366a1476cb5, store: [table=TestAcidGuarantees family=A region=94f3ccf9f8a638fa974e1f69627832ef] 2024-11-24T22:15:42,673 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bd6592c37bb2:0-0 {event_type=RS_FLUSH_REGIONS, pid=108}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/94f3ccf9f8a638fa974e1f69627832ef/.tmp/A/8d09d43826484065b9293366a1476cb5 is 175, key is test_row_0/A:col10/1732486541712/Put/seqid=0 2024-11-24T22:15:42,675 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45929 is added to blk_1073742253_1429 (size=31155) 2024-11-24T22:15:42,677 INFO [RS_FLUSH_OPERATIONS-regionserver/bd6592c37bb2:0-0 {event_type=RS_FLUSH_REGIONS, pid=108}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=259, memsize=22.4 K, hasBloomFilter=true, into tmp file hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/94f3ccf9f8a638fa974e1f69627832ef/.tmp/A/8d09d43826484065b9293366a1476cb5 2024-11-24T22:15:42,684 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bd6592c37bb2:0-0 {event_type=RS_FLUSH_REGIONS, pid=108}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/94f3ccf9f8a638fa974e1f69627832ef/.tmp/B/154e2a22f31844d198f56aa1a33d20ca is 50, key is test_row_0/B:col10/1732486541712/Put/seqid=0 2024-11-24T22:15:42,686 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45929 is added to blk_1073742254_1430 (size=12201) 2024-11-24T22:15:42,687 INFO [RS_FLUSH_OPERATIONS-regionserver/bd6592c37bb2:0-0 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=259 (bloomFilter=true), to=hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/94f3ccf9f8a638fa974e1f69627832ef/.tmp/B/154e2a22f31844d198f56aa1a33d20ca 2024-11-24T22:15:42,693 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bd6592c37bb2:0-0 {event_type=RS_FLUSH_REGIONS, pid=108}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/94f3ccf9f8a638fa974e1f69627832ef/.tmp/C/73ea5ac0072e48f180abd4ef66e7d3fe is 50, key is test_row_0/C:col10/1732486541712/Put/seqid=0 2024-11-24T22:15:42,696 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45929 is added to blk_1073742255_1431 (size=12201) 2024-11-24T22:15:42,821 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37701 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=107 2024-11-24T22:15:42,896 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40087 {}] regionserver.HRegion(8855): Flush requested on 94f3ccf9f8a638fa974e1f69627832ef 2024-11-24T22:15:42,896 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732486521610.94f3ccf9f8a638fa974e1f69627832ef. as already flushing 2024-11-24T22:15:42,952 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40087 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94f3ccf9f8a638fa974e1f69627832ef, server=bd6592c37bb2,40087,1732486423713 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-24T22:15:42,952 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40087 {}] ipc.CallRunner(138): callId: 99 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:32846 deadline: 1732486602938, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94f3ccf9f8a638fa974e1f69627832ef, server=bd6592c37bb2,40087,1732486423713 2024-11-24T22:15:42,953 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40087 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94f3ccf9f8a638fa974e1f69627832ef, server=bd6592c37bb2,40087,1732486423713 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-24T22:15:42,953 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40087 {}] ipc.CallRunner(138): callId: 83 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:32832 deadline: 1732486602942, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94f3ccf9f8a638fa974e1f69627832ef, server=bd6592c37bb2,40087,1732486423713 2024-11-24T22:15:42,953 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732486521610.94f3ccf9f8a638fa974e1f69627832ef., hostname=bd6592c37bb2,40087,1732486423713, seqNum=5 , the old value is region=TestAcidGuarantees,,1732486521610.94f3ccf9f8a638fa974e1f69627832ef., hostname=bd6592c37bb2,40087,1732486423713, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94f3ccf9f8a638fa974e1f69627832ef, server=bd6592c37bb2,40087,1732486423713 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T22:15:42,953 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732486521610.94f3ccf9f8a638fa974e1f69627832ef., hostname=bd6592c37bb2,40087,1732486423713, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94f3ccf9f8a638fa974e1f69627832ef, server=bd6592c37bb2,40087,1732486423713 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T22:15:42,953 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732486521610.94f3ccf9f8a638fa974e1f69627832ef., hostname=bd6592c37bb2,40087,1732486423713, seqNum=5 because the exception is null or not the one we care about 2024-11-24T22:15:42,953 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732486521610.94f3ccf9f8a638fa974e1f69627832ef., hostname=bd6592c37bb2,40087,1732486423713, seqNum=5 , the old value is region=TestAcidGuarantees,,1732486521610.94f3ccf9f8a638fa974e1f69627832ef., hostname=bd6592c37bb2,40087,1732486423713, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94f3ccf9f8a638fa974e1f69627832ef, server=bd6592c37bb2,40087,1732486423713 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T22:15:42,953 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732486521610.94f3ccf9f8a638fa974e1f69627832ef., hostname=bd6592c37bb2,40087,1732486423713, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94f3ccf9f8a638fa974e1f69627832ef, server=bd6592c37bb2,40087,1732486423713 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T22:15:42,953 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732486521610.94f3ccf9f8a638fa974e1f69627832ef., hostname=bd6592c37bb2,40087,1732486423713, seqNum=5 because the exception is null or not the one we care about 2024-11-24T22:15:42,956 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40087 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94f3ccf9f8a638fa974e1f69627832ef, server=bd6592c37bb2,40087,1732486423713 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-24T22:15:42,957 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40087 {}] ipc.CallRunner(138): callId: 101 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:32774 deadline: 1732486602943, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94f3ccf9f8a638fa974e1f69627832ef, server=bd6592c37bb2,40087,1732486423713 2024-11-24T22:15:42,957 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732486521610.94f3ccf9f8a638fa974e1f69627832ef., hostname=bd6592c37bb2,40087,1732486423713, seqNum=5 , the old value is region=TestAcidGuarantees,,1732486521610.94f3ccf9f8a638fa974e1f69627832ef., hostname=bd6592c37bb2,40087,1732486423713, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94f3ccf9f8a638fa974e1f69627832ef, server=bd6592c37bb2,40087,1732486423713 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T22:15:42,957 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732486521610.94f3ccf9f8a638fa974e1f69627832ef., hostname=bd6592c37bb2,40087,1732486423713, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94f3ccf9f8a638fa974e1f69627832ef, server=bd6592c37bb2,40087,1732486423713 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T22:15:42,957 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732486521610.94f3ccf9f8a638fa974e1f69627832ef., hostname=bd6592c37bb2,40087,1732486423713, seqNum=5 because the exception is null or not the one we care about 2024-11-24T22:15:42,970 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40087 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94f3ccf9f8a638fa974e1f69627832ef, server=bd6592c37bb2,40087,1732486423713 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-24T22:15:42,970 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40087 {}] ipc.CallRunner(138): callId: 62 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:32810 deadline: 1732486602961, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94f3ccf9f8a638fa974e1f69627832ef, server=bd6592c37bb2,40087,1732486423713 2024-11-24T22:15:42,971 WARN [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRpcRetryingCaller(177): Call to bd6592c37bb2,40087,1732486423713 for 'test_row_2' in 94f3ccf9f8a638fa974e1f69627832ef of TestAcidGuarantees failed, tries = 8, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 8248 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94f3ccf9f8a638fa974e1f69627832ef, server=bd6592c37bb2,40087,1732486423713 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor44.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94f3ccf9f8a638fa974e1f69627832ef, server=bd6592c37bb2,40087,1732486423713 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-24T22:15:42,971 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732486521610.94f3ccf9f8a638fa974e1f69627832ef., hostname=bd6592c37bb2,40087,1732486423713, seqNum=5 , the old value is region=TestAcidGuarantees,,1732486521610.94f3ccf9f8a638fa974e1f69627832ef., hostname=bd6592c37bb2,40087,1732486423713, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94f3ccf9f8a638fa974e1f69627832ef, server=bd6592c37bb2,40087,1732486423713 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T22:15:42,971 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732486521610.94f3ccf9f8a638fa974e1f69627832ef., hostname=bd6592c37bb2,40087,1732486423713, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94f3ccf9f8a638fa974e1f69627832ef, server=bd6592c37bb2,40087,1732486423713 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T22:15:42,971 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732486521610.94f3ccf9f8a638fa974e1f69627832ef., hostname=bd6592c37bb2,40087,1732486423713, seqNum=5 because the exception is null or not the one we care about 2024-11-24T22:15:43,070 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40087 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94f3ccf9f8a638fa974e1f69627832ef, server=bd6592c37bb2,40087,1732486423713 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-24T22:15:43,070 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40087 {}] ipc.CallRunner(138): callId: 100 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:32846 deadline: 1732486603061, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94f3ccf9f8a638fa974e1f69627832ef, server=bd6592c37bb2,40087,1732486423713 2024-11-24T22:15:43,070 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40087 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94f3ccf9f8a638fa974e1f69627832ef, server=bd6592c37bb2,40087,1732486423713 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-24T22:15:43,070 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732486521610.94f3ccf9f8a638fa974e1f69627832ef., hostname=bd6592c37bb2,40087,1732486423713, seqNum=5 , the old value is region=TestAcidGuarantees,,1732486521610.94f3ccf9f8a638fa974e1f69627832ef., hostname=bd6592c37bb2,40087,1732486423713, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94f3ccf9f8a638fa974e1f69627832ef, server=bd6592c37bb2,40087,1732486423713 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T22:15:43,070 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732486521610.94f3ccf9f8a638fa974e1f69627832ef., hostname=bd6592c37bb2,40087,1732486423713, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94f3ccf9f8a638fa974e1f69627832ef, server=bd6592c37bb2,40087,1732486423713 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T22:15:43,070 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40087 {}] ipc.CallRunner(138): callId: 84 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:32832 deadline: 1732486603061, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94f3ccf9f8a638fa974e1f69627832ef, server=bd6592c37bb2,40087,1732486423713 2024-11-24T22:15:43,070 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732486521610.94f3ccf9f8a638fa974e1f69627832ef., hostname=bd6592c37bb2,40087,1732486423713, seqNum=5 because the exception is null or not the one we care about 2024-11-24T22:15:43,070 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732486521610.94f3ccf9f8a638fa974e1f69627832ef., hostname=bd6592c37bb2,40087,1732486423713, seqNum=5 , the old value is region=TestAcidGuarantees,,1732486521610.94f3ccf9f8a638fa974e1f69627832ef., hostname=bd6592c37bb2,40087,1732486423713, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94f3ccf9f8a638fa974e1f69627832ef, server=bd6592c37bb2,40087,1732486423713 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T22:15:43,070 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40087 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94f3ccf9f8a638fa974e1f69627832ef, server=bd6592c37bb2,40087,1732486423713 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-24T22:15:43,070 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732486521610.94f3ccf9f8a638fa974e1f69627832ef., hostname=bd6592c37bb2,40087,1732486423713, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94f3ccf9f8a638fa974e1f69627832ef, server=bd6592c37bb2,40087,1732486423713 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T22:15:43,071 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732486521610.94f3ccf9f8a638fa974e1f69627832ef., hostname=bd6592c37bb2,40087,1732486423713, seqNum=5 because the exception is null or not the one we care about 2024-11-24T22:15:43,071 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40087 {}] ipc.CallRunner(138): callId: 102 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:32774 deadline: 1732486603061, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94f3ccf9f8a638fa974e1f69627832ef, server=bd6592c37bb2,40087,1732486423713 2024-11-24T22:15:43,071 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732486521610.94f3ccf9f8a638fa974e1f69627832ef., hostname=bd6592c37bb2,40087,1732486423713, seqNum=5 , the old value is region=TestAcidGuarantees,,1732486521610.94f3ccf9f8a638fa974e1f69627832ef., hostname=bd6592c37bb2,40087,1732486423713, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94f3ccf9f8a638fa974e1f69627832ef, server=bd6592c37bb2,40087,1732486423713 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T22:15:43,071 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732486521610.94f3ccf9f8a638fa974e1f69627832ef., hostname=bd6592c37bb2,40087,1732486423713, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94f3ccf9f8a638fa974e1f69627832ef, server=bd6592c37bb2,40087,1732486423713 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T22:15:43,071 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732486521610.94f3ccf9f8a638fa974e1f69627832ef., hostname=bd6592c37bb2,40087,1732486423713, seqNum=5 because the exception is null or not the one we care about 2024-11-24T22:15:43,096 INFO [RS_FLUSH_OPERATIONS-regionserver/bd6592c37bb2:0-0 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=259 (bloomFilter=true), to=hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/94f3ccf9f8a638fa974e1f69627832ef/.tmp/C/73ea5ac0072e48f180abd4ef66e7d3fe 2024-11-24T22:15:43,100 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bd6592c37bb2:0-0 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/94f3ccf9f8a638fa974e1f69627832ef/.tmp/A/8d09d43826484065b9293366a1476cb5 as hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/94f3ccf9f8a638fa974e1f69627832ef/A/8d09d43826484065b9293366a1476cb5 2024-11-24T22:15:43,104 INFO [RS_FLUSH_OPERATIONS-regionserver/bd6592c37bb2:0-0 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/94f3ccf9f8a638fa974e1f69627832ef/A/8d09d43826484065b9293366a1476cb5, entries=150, sequenceid=259, filesize=30.4 K 2024-11-24T22:15:43,104 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bd6592c37bb2:0-0 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/94f3ccf9f8a638fa974e1f69627832ef/.tmp/B/154e2a22f31844d198f56aa1a33d20ca as hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/94f3ccf9f8a638fa974e1f69627832ef/B/154e2a22f31844d198f56aa1a33d20ca 2024-11-24T22:15:43,107 INFO [RS_FLUSH_OPERATIONS-regionserver/bd6592c37bb2:0-0 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/94f3ccf9f8a638fa974e1f69627832ef/B/154e2a22f31844d198f56aa1a33d20ca, entries=150, sequenceid=259, filesize=11.9 K 2024-11-24T22:15:43,108 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bd6592c37bb2:0-0 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/94f3ccf9f8a638fa974e1f69627832ef/.tmp/C/73ea5ac0072e48f180abd4ef66e7d3fe as hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/94f3ccf9f8a638fa974e1f69627832ef/C/73ea5ac0072e48f180abd4ef66e7d3fe 2024-11-24T22:15:43,111 INFO [RS_FLUSH_OPERATIONS-regionserver/bd6592c37bb2:0-0 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/94f3ccf9f8a638fa974e1f69627832ef/C/73ea5ac0072e48f180abd4ef66e7d3fe, entries=150, sequenceid=259, filesize=11.9 K 2024-11-24T22:15:43,112 INFO [RS_FLUSH_OPERATIONS-regionserver/bd6592c37bb2:0-0 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.HRegion(3140): Finished flush of dataSize ~67.09 KB/68700, heapSize ~199.92 KB/204720, currentSize=107.34 KB/109920 for 94f3ccf9f8a638fa974e1f69627832ef in 455ms, sequenceid=259, compaction requested=true 2024-11-24T22:15:43,112 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bd6592c37bb2:0-0 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.HRegion(2603): Flush status journal for 94f3ccf9f8a638fa974e1f69627832ef: 2024-11-24T22:15:43,112 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bd6592c37bb2:0-0 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732486521610.94f3ccf9f8a638fa974e1f69627832ef. 2024-11-24T22:15:43,112 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bd6592c37bb2:0-0 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=108 2024-11-24T22:15:43,113 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37701 {}] master.HMaster(4169): Remote procedure done, pid=108 2024-11-24T22:15:43,115 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=108, resume processing ppid=107 2024-11-24T22:15:43,115 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=108, ppid=107, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 608 msec 2024-11-24T22:15:43,116 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=107, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=107, table=TestAcidGuarantees in 614 msec 2024-11-24T22:15:43,131 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37701 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=107 2024-11-24T22:15:43,131 INFO [RPCClient-NioEventLoopGroup-4-8 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-24T22:15:43,131 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37701 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-24T22:15:43,132 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37701 {}] procedure2.ProcedureExecutor(1139): Stored pid=109, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=109, table=TestAcidGuarantees 2024-11-24T22:15:43,132 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37701 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=109 2024-11-24T22:15:43,133 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=109, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=109, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-24T22:15:43,133 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=109, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=109, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-24T22:15:43,133 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=110, ppid=109, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-24T22:15:43,241 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37701 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=109 2024-11-24T22:15:43,285 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=40087 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=110 2024-11-24T22:15:43,285 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bd6592c37bb2:0-1 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732486521610.94f3ccf9f8a638fa974e1f69627832ef. 2024-11-24T22:15:43,285 INFO [RS_FLUSH_OPERATIONS-regionserver/bd6592c37bb2:0-1 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.HRegion(2902): Flushing 94f3ccf9f8a638fa974e1f69627832ef 3/3 column families, dataSize=107.34 KB heapSize=319.50 KB 2024-11-24T22:15:43,286 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bd6592c37bb2:0-1 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 94f3ccf9f8a638fa974e1f69627832ef, store=A 2024-11-24T22:15:43,286 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bd6592c37bb2:0-1 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-24T22:15:43,286 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bd6592c37bb2:0-1 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 94f3ccf9f8a638fa974e1f69627832ef, store=B 2024-11-24T22:15:43,286 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bd6592c37bb2:0-1 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-24T22:15:43,286 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bd6592c37bb2:0-1 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 94f3ccf9f8a638fa974e1f69627832ef, store=C 2024-11-24T22:15:43,286 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bd6592c37bb2:0-1 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-24T22:15:43,288 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40087 {}] regionserver.HRegion(8855): Flush requested on 94f3ccf9f8a638fa974e1f69627832ef 2024-11-24T22:15:43,288 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732486521610.94f3ccf9f8a638fa974e1f69627832ef. as already flushing 2024-11-24T22:15:43,293 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bd6592c37bb2:0-1 {event_type=RS_FLUSH_REGIONS, pid=110}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241124939393c18b524edc9db85217878b0130_94f3ccf9f8a638fa974e1f69627832ef is 50, key is test_row_0/A:col10/1732486542936/Put/seqid=0 2024-11-24T22:15:43,296 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45929 is added to blk_1073742256_1432 (size=12454) 2024-11-24T22:15:43,366 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40087 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94f3ccf9f8a638fa974e1f69627832ef, server=bd6592c37bb2,40087,1732486423713 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-24T22:15:43,366 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40087 {}] ipc.CallRunner(138): callId: 88 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:32832 deadline: 1732486603347, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94f3ccf9f8a638fa974e1f69627832ef, server=bd6592c37bb2,40087,1732486423713 2024-11-24T22:15:43,367 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732486521610.94f3ccf9f8a638fa974e1f69627832ef., hostname=bd6592c37bb2,40087,1732486423713, seqNum=5 , the old value is region=TestAcidGuarantees,,1732486521610.94f3ccf9f8a638fa974e1f69627832ef., hostname=bd6592c37bb2,40087,1732486423713, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94f3ccf9f8a638fa974e1f69627832ef, server=bd6592c37bb2,40087,1732486423713 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T22:15:43,367 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40087 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94f3ccf9f8a638fa974e1f69627832ef, server=bd6592c37bb2,40087,1732486423713 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-24T22:15:43,367 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732486521610.94f3ccf9f8a638fa974e1f69627832ef., hostname=bd6592c37bb2,40087,1732486423713, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94f3ccf9f8a638fa974e1f69627832ef, server=bd6592c37bb2,40087,1732486423713 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T22:15:43,367 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732486521610.94f3ccf9f8a638fa974e1f69627832ef., hostname=bd6592c37bb2,40087,1732486423713, seqNum=5 because the exception is null or not the one we care about 2024-11-24T22:15:43,367 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40087 {}] ipc.CallRunner(138): callId: 105 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:32846 deadline: 1732486603347, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94f3ccf9f8a638fa974e1f69627832ef, server=bd6592c37bb2,40087,1732486423713 2024-11-24T22:15:43,367 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732486521610.94f3ccf9f8a638fa974e1f69627832ef., hostname=bd6592c37bb2,40087,1732486423713, seqNum=5 , the old value is region=TestAcidGuarantees,,1732486521610.94f3ccf9f8a638fa974e1f69627832ef., hostname=bd6592c37bb2,40087,1732486423713, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94f3ccf9f8a638fa974e1f69627832ef, server=bd6592c37bb2,40087,1732486423713 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T22:15:43,367 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732486521610.94f3ccf9f8a638fa974e1f69627832ef., hostname=bd6592c37bb2,40087,1732486423713, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94f3ccf9f8a638fa974e1f69627832ef, server=bd6592c37bb2,40087,1732486423713 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T22:15:43,367 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40087 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94f3ccf9f8a638fa974e1f69627832ef, server=bd6592c37bb2,40087,1732486423713 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-24T22:15:43,367 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732486521610.94f3ccf9f8a638fa974e1f69627832ef., hostname=bd6592c37bb2,40087,1732486423713, seqNum=5 because the exception is null or not the one we care about 2024-11-24T22:15:43,367 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40087 {}] ipc.CallRunner(138): callId: 106 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:32774 deadline: 1732486603347, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94f3ccf9f8a638fa974e1f69627832ef, server=bd6592c37bb2,40087,1732486423713 2024-11-24T22:15:43,367 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732486521610.94f3ccf9f8a638fa974e1f69627832ef., hostname=bd6592c37bb2,40087,1732486423713, seqNum=5 , the old value is region=TestAcidGuarantees,,1732486521610.94f3ccf9f8a638fa974e1f69627832ef., hostname=bd6592c37bb2,40087,1732486423713, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94f3ccf9f8a638fa974e1f69627832ef, server=bd6592c37bb2,40087,1732486423713 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T22:15:43,367 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732486521610.94f3ccf9f8a638fa974e1f69627832ef., hostname=bd6592c37bb2,40087,1732486423713, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94f3ccf9f8a638fa974e1f69627832ef, server=bd6592c37bb2,40087,1732486423713 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T22:15:43,368 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732486521610.94f3ccf9f8a638fa974e1f69627832ef., hostname=bd6592c37bb2,40087,1732486423713, seqNum=5 because the exception is null or not the one we care about 2024-11-24T22:15:43,451 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37701 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=109 2024-11-24T22:15:43,482 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40087 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94f3ccf9f8a638fa974e1f69627832ef, server=bd6592c37bb2,40087,1732486423713 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-24T22:15:43,482 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40087 {}] ipc.CallRunner(138): callId: 89 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:32832 deadline: 1732486603471, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94f3ccf9f8a638fa974e1f69627832ef, server=bd6592c37bb2,40087,1732486423713 2024-11-24T22:15:43,483 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40087 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94f3ccf9f8a638fa974e1f69627832ef, server=bd6592c37bb2,40087,1732486423713 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-24T22:15:43,483 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732486521610.94f3ccf9f8a638fa974e1f69627832ef., hostname=bd6592c37bb2,40087,1732486423713, seqNum=5 , the old value is region=TestAcidGuarantees,,1732486521610.94f3ccf9f8a638fa974e1f69627832ef., hostname=bd6592c37bb2,40087,1732486423713, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94f3ccf9f8a638fa974e1f69627832ef, server=bd6592c37bb2,40087,1732486423713 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T22:15:43,483 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40087 {}] ipc.CallRunner(138): callId: 106 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:32846 deadline: 1732486603471, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94f3ccf9f8a638fa974e1f69627832ef, server=bd6592c37bb2,40087,1732486423713 2024-11-24T22:15:43,483 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732486521610.94f3ccf9f8a638fa974e1f69627832ef., hostname=bd6592c37bb2,40087,1732486423713, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94f3ccf9f8a638fa974e1f69627832ef, server=bd6592c37bb2,40087,1732486423713 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T22:15:43,483 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732486521610.94f3ccf9f8a638fa974e1f69627832ef., hostname=bd6592c37bb2,40087,1732486423713, seqNum=5 because the exception is null or not the one we care about 2024-11-24T22:15:43,483 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40087 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94f3ccf9f8a638fa974e1f69627832ef, server=bd6592c37bb2,40087,1732486423713 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-24T22:15:43,483 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732486521610.94f3ccf9f8a638fa974e1f69627832ef., hostname=bd6592c37bb2,40087,1732486423713, seqNum=5 , the old value is region=TestAcidGuarantees,,1732486521610.94f3ccf9f8a638fa974e1f69627832ef., hostname=bd6592c37bb2,40087,1732486423713, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94f3ccf9f8a638fa974e1f69627832ef, server=bd6592c37bb2,40087,1732486423713 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T22:15:43,483 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40087 {}] ipc.CallRunner(138): callId: 107 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:32774 deadline: 1732486603471, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94f3ccf9f8a638fa974e1f69627832ef, server=bd6592c37bb2,40087,1732486423713 2024-11-24T22:15:43,483 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732486521610.94f3ccf9f8a638fa974e1f69627832ef., hostname=bd6592c37bb2,40087,1732486423713, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94f3ccf9f8a638fa974e1f69627832ef, server=bd6592c37bb2,40087,1732486423713 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T22:15:43,483 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732486521610.94f3ccf9f8a638fa974e1f69627832ef., hostname=bd6592c37bb2,40087,1732486423713, seqNum=5 because the exception is null or not the one we care about 2024-11-24T22:15:43,483 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732486521610.94f3ccf9f8a638fa974e1f69627832ef., hostname=bd6592c37bb2,40087,1732486423713, seqNum=5 , the old value is region=TestAcidGuarantees,,1732486521610.94f3ccf9f8a638fa974e1f69627832ef., hostname=bd6592c37bb2,40087,1732486423713, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94f3ccf9f8a638fa974e1f69627832ef, server=bd6592c37bb2,40087,1732486423713 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T22:15:43,483 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732486521610.94f3ccf9f8a638fa974e1f69627832ef., hostname=bd6592c37bb2,40087,1732486423713, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94f3ccf9f8a638fa974e1f69627832ef, server=bd6592c37bb2,40087,1732486423713 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T22:15:43,483 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732486521610.94f3ccf9f8a638fa974e1f69627832ef., hostname=bd6592c37bb2,40087,1732486423713, seqNum=5 because the exception is null or not the one we care about 2024-11-24T22:15:43,697 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bd6592c37bb2:0-1 {event_type=RS_FLUSH_REGIONS, pid=110}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T22:15:43,701 INFO [RS_FLUSH_OPERATIONS-regionserver/bd6592c37bb2:0-1 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241124939393c18b524edc9db85217878b0130_94f3ccf9f8a638fa974e1f69627832ef to hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241124939393c18b524edc9db85217878b0130_94f3ccf9f8a638fa974e1f69627832ef 2024-11-24T22:15:43,701 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bd6592c37bb2:0-1 {event_type=RS_FLUSH_REGIONS, pid=110}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/94f3ccf9f8a638fa974e1f69627832ef/.tmp/A/19c3184bc0c8467ea8a35a00f8b1bd84, store: [table=TestAcidGuarantees family=A region=94f3ccf9f8a638fa974e1f69627832ef] 2024-11-24T22:15:43,702 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bd6592c37bb2:0-1 {event_type=RS_FLUSH_REGIONS, pid=110}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/94f3ccf9f8a638fa974e1f69627832ef/.tmp/A/19c3184bc0c8467ea8a35a00f8b1bd84 is 175, key is test_row_0/A:col10/1732486542936/Put/seqid=0 2024-11-24T22:15:43,704 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40087 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94f3ccf9f8a638fa974e1f69627832ef, server=bd6592c37bb2,40087,1732486423713 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-24T22:15:43,705 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40087 {}] ipc.CallRunner(138): callId: 90 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:32832 deadline: 1732486603691, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94f3ccf9f8a638fa974e1f69627832ef, server=bd6592c37bb2,40087,1732486423713 2024-11-24T22:15:43,705 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40087 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94f3ccf9f8a638fa974e1f69627832ef, server=bd6592c37bb2,40087,1732486423713 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-24T22:15:43,705 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732486521610.94f3ccf9f8a638fa974e1f69627832ef., hostname=bd6592c37bb2,40087,1732486423713, seqNum=5 , the old value is region=TestAcidGuarantees,,1732486521610.94f3ccf9f8a638fa974e1f69627832ef., hostname=bd6592c37bb2,40087,1732486423713, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94f3ccf9f8a638fa974e1f69627832ef, server=bd6592c37bb2,40087,1732486423713 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T22:15:43,705 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40087 {}] ipc.CallRunner(138): callId: 107 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:32846 deadline: 1732486603691, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94f3ccf9f8a638fa974e1f69627832ef, server=bd6592c37bb2,40087,1732486423713 2024-11-24T22:15:43,705 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732486521610.94f3ccf9f8a638fa974e1f69627832ef., hostname=bd6592c37bb2,40087,1732486423713, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94f3ccf9f8a638fa974e1f69627832ef, server=bd6592c37bb2,40087,1732486423713 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T22:15:43,705 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732486521610.94f3ccf9f8a638fa974e1f69627832ef., hostname=bd6592c37bb2,40087,1732486423713, seqNum=5 because the exception is null or not the one we care about 2024-11-24T22:15:43,705 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45929 is added to blk_1073742257_1433 (size=31255) 2024-11-24T22:15:43,705 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732486521610.94f3ccf9f8a638fa974e1f69627832ef., hostname=bd6592c37bb2,40087,1732486423713, seqNum=5 , the old value is region=TestAcidGuarantees,,1732486521610.94f3ccf9f8a638fa974e1f69627832ef., hostname=bd6592c37bb2,40087,1732486423713, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94f3ccf9f8a638fa974e1f69627832ef, server=bd6592c37bb2,40087,1732486423713 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T22:15:43,705 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40087 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94f3ccf9f8a638fa974e1f69627832ef, server=bd6592c37bb2,40087,1732486423713 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-24T22:15:43,705 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732486521610.94f3ccf9f8a638fa974e1f69627832ef., hostname=bd6592c37bb2,40087,1732486423713, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94f3ccf9f8a638fa974e1f69627832ef, server=bd6592c37bb2,40087,1732486423713 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T22:15:43,705 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732486521610.94f3ccf9f8a638fa974e1f69627832ef., hostname=bd6592c37bb2,40087,1732486423713, seqNum=5 because the exception is null or not the one we care about 2024-11-24T22:15:43,705 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40087 {}] ipc.CallRunner(138): callId: 108 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:32774 deadline: 1732486603691, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94f3ccf9f8a638fa974e1f69627832ef, server=bd6592c37bb2,40087,1732486423713 2024-11-24T22:15:43,706 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732486521610.94f3ccf9f8a638fa974e1f69627832ef., hostname=bd6592c37bb2,40087,1732486423713, seqNum=5 , the old value is region=TestAcidGuarantees,,1732486521610.94f3ccf9f8a638fa974e1f69627832ef., hostname=bd6592c37bb2,40087,1732486423713, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94f3ccf9f8a638fa974e1f69627832ef, server=bd6592c37bb2,40087,1732486423713 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T22:15:43,706 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732486521610.94f3ccf9f8a638fa974e1f69627832ef., hostname=bd6592c37bb2,40087,1732486423713, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94f3ccf9f8a638fa974e1f69627832ef, server=bd6592c37bb2,40087,1732486423713 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T22:15:43,706 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732486521610.94f3ccf9f8a638fa974e1f69627832ef., hostname=bd6592c37bb2,40087,1732486423713, seqNum=5 because the exception is null or not the one we care about 2024-11-24T22:15:43,707 INFO [RS_FLUSH_OPERATIONS-regionserver/bd6592c37bb2:0-1 {event_type=RS_FLUSH_REGIONS, pid=110}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=278, memsize=35.8 K, hasBloomFilter=true, into tmp file hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/94f3ccf9f8a638fa974e1f69627832ef/.tmp/A/19c3184bc0c8467ea8a35a00f8b1bd84 2024-11-24T22:15:43,713 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bd6592c37bb2:0-1 {event_type=RS_FLUSH_REGIONS, pid=110}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/94f3ccf9f8a638fa974e1f69627832ef/.tmp/B/472d5538cdba45b3ae87ece46ad44a5e is 50, key is test_row_0/B:col10/1732486542936/Put/seqid=0 2024-11-24T22:15:43,717 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45929 is added to blk_1073742258_1434 (size=12301) 2024-11-24T22:15:43,717 INFO [RS_FLUSH_OPERATIONS-regionserver/bd6592c37bb2:0-1 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=35.78 KB at sequenceid=278 (bloomFilter=true), to=hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/94f3ccf9f8a638fa974e1f69627832ef/.tmp/B/472d5538cdba45b3ae87ece46ad44a5e 2024-11-24T22:15:43,725 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bd6592c37bb2:0-1 {event_type=RS_FLUSH_REGIONS, pid=110}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/94f3ccf9f8a638fa974e1f69627832ef/.tmp/C/4920a0a11aed4fdfbbaa49f3afcc84b1 is 50, key is test_row_0/C:col10/1732486542936/Put/seqid=0 2024-11-24T22:15:43,728 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45929 is added to blk_1073742259_1435 (size=12301) 2024-11-24T22:15:43,761 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37701 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=109 2024-11-24T22:15:43,994 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40087 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94f3ccf9f8a638fa974e1f69627832ef, server=bd6592c37bb2,40087,1732486423713 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-24T22:15:43,994 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40087 {}] ipc.CallRunner(138): callId: 77 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:32822 deadline: 1732486603980, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94f3ccf9f8a638fa974e1f69627832ef, server=bd6592c37bb2,40087,1732486423713 2024-11-24T22:15:43,994 WARN [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRpcRetryingCaller(177): Call to bd6592c37bb2,40087,1732486423713 for 'test_row_2' in 94f3ccf9f8a638fa974e1f69627832ef of TestAcidGuarantees failed, tries = 8, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 8253 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94f3ccf9f8a638fa974e1f69627832ef, server=bd6592c37bb2,40087,1732486423713 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor44.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94f3ccf9f8a638fa974e1f69627832ef, server=bd6592c37bb2,40087,1732486423713 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-24T22:15:43,995 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732486521610.94f3ccf9f8a638fa974e1f69627832ef., hostname=bd6592c37bb2,40087,1732486423713, seqNum=5 , the old value is region=TestAcidGuarantees,,1732486521610.94f3ccf9f8a638fa974e1f69627832ef., hostname=bd6592c37bb2,40087,1732486423713, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94f3ccf9f8a638fa974e1f69627832ef, server=bd6592c37bb2,40087,1732486423713 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T22:15:43,995 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732486521610.94f3ccf9f8a638fa974e1f69627832ef., hostname=bd6592c37bb2,40087,1732486423713, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94f3ccf9f8a638fa974e1f69627832ef, server=bd6592c37bb2,40087,1732486423713 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T22:15:43,995 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732486521610.94f3ccf9f8a638fa974e1f69627832ef., hostname=bd6592c37bb2,40087,1732486423713, seqNum=5 because the exception is null or not the one we care about 2024-11-24T22:15:44,023 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40087 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94f3ccf9f8a638fa974e1f69627832ef, server=bd6592c37bb2,40087,1732486423713 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-24T22:15:44,023 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40087 {}] ipc.CallRunner(138): callId: 91 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:32832 deadline: 1732486604011, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94f3ccf9f8a638fa974e1f69627832ef, server=bd6592c37bb2,40087,1732486423713 2024-11-24T22:15:44,023 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40087 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94f3ccf9f8a638fa974e1f69627832ef, server=bd6592c37bb2,40087,1732486423713 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-24T22:15:44,023 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732486521610.94f3ccf9f8a638fa974e1f69627832ef., hostname=bd6592c37bb2,40087,1732486423713, seqNum=5 , the old value is region=TestAcidGuarantees,,1732486521610.94f3ccf9f8a638fa974e1f69627832ef., hostname=bd6592c37bb2,40087,1732486423713, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94f3ccf9f8a638fa974e1f69627832ef, server=bd6592c37bb2,40087,1732486423713 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T22:15:44,023 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40087 {}] ipc.CallRunner(138): callId: 108 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:32846 deadline: 1732486604011, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94f3ccf9f8a638fa974e1f69627832ef, server=bd6592c37bb2,40087,1732486423713 2024-11-24T22:15:44,023 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732486521610.94f3ccf9f8a638fa974e1f69627832ef., hostname=bd6592c37bb2,40087,1732486423713, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94f3ccf9f8a638fa974e1f69627832ef, server=bd6592c37bb2,40087,1732486423713 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T22:15:44,023 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732486521610.94f3ccf9f8a638fa974e1f69627832ef., hostname=bd6592c37bb2,40087,1732486423713, seqNum=5 because the exception is null or not the one we care about 2024-11-24T22:15:44,023 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40087 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94f3ccf9f8a638fa974e1f69627832ef, server=bd6592c37bb2,40087,1732486423713 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-24T22:15:44,023 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732486521610.94f3ccf9f8a638fa974e1f69627832ef., hostname=bd6592c37bb2,40087,1732486423713, seqNum=5 , the old value is region=TestAcidGuarantees,,1732486521610.94f3ccf9f8a638fa974e1f69627832ef., hostname=bd6592c37bb2,40087,1732486423713, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94f3ccf9f8a638fa974e1f69627832ef, server=bd6592c37bb2,40087,1732486423713 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T22:15:44,024 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40087 {}] ipc.CallRunner(138): callId: 109 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:32774 deadline: 1732486604011, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94f3ccf9f8a638fa974e1f69627832ef, server=bd6592c37bb2,40087,1732486423713 2024-11-24T22:15:44,024 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732486521610.94f3ccf9f8a638fa974e1f69627832ef., hostname=bd6592c37bb2,40087,1732486423713, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94f3ccf9f8a638fa974e1f69627832ef, server=bd6592c37bb2,40087,1732486423713 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T22:15:44,024 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732486521610.94f3ccf9f8a638fa974e1f69627832ef., hostname=bd6592c37bb2,40087,1732486423713, seqNum=5 because the exception is null or not the one we care about 2024-11-24T22:15:44,024 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732486521610.94f3ccf9f8a638fa974e1f69627832ef., hostname=bd6592c37bb2,40087,1732486423713, seqNum=5 , the old value is region=TestAcidGuarantees,,1732486521610.94f3ccf9f8a638fa974e1f69627832ef., hostname=bd6592c37bb2,40087,1732486423713, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94f3ccf9f8a638fa974e1f69627832ef, server=bd6592c37bb2,40087,1732486423713 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T22:15:44,024 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732486521610.94f3ccf9f8a638fa974e1f69627832ef., hostname=bd6592c37bb2,40087,1732486423713, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94f3ccf9f8a638fa974e1f69627832ef, server=bd6592c37bb2,40087,1732486423713 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T22:15:44,024 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732486521610.94f3ccf9f8a638fa974e1f69627832ef., hostname=bd6592c37bb2,40087,1732486423713, seqNum=5 because the exception is null or not the one we care about 2024-11-24T22:15:44,128 INFO [RS_FLUSH_OPERATIONS-regionserver/bd6592c37bb2:0-1 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=35.78 KB at sequenceid=278 (bloomFilter=true), to=hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/94f3ccf9f8a638fa974e1f69627832ef/.tmp/C/4920a0a11aed4fdfbbaa49f3afcc84b1 2024-11-24T22:15:44,134 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bd6592c37bb2:0-1 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/94f3ccf9f8a638fa974e1f69627832ef/.tmp/A/19c3184bc0c8467ea8a35a00f8b1bd84 as hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/94f3ccf9f8a638fa974e1f69627832ef/A/19c3184bc0c8467ea8a35a00f8b1bd84 2024-11-24T22:15:44,138 INFO [RS_FLUSH_OPERATIONS-regionserver/bd6592c37bb2:0-1 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/94f3ccf9f8a638fa974e1f69627832ef/A/19c3184bc0c8467ea8a35a00f8b1bd84, entries=150, sequenceid=278, filesize=30.5 K 2024-11-24T22:15:44,139 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bd6592c37bb2:0-1 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/94f3ccf9f8a638fa974e1f69627832ef/.tmp/B/472d5538cdba45b3ae87ece46ad44a5e as hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/94f3ccf9f8a638fa974e1f69627832ef/B/472d5538cdba45b3ae87ece46ad44a5e 2024-11-24T22:15:44,142 INFO [RS_FLUSH_OPERATIONS-regionserver/bd6592c37bb2:0-1 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/94f3ccf9f8a638fa974e1f69627832ef/B/472d5538cdba45b3ae87ece46ad44a5e, entries=150, sequenceid=278, filesize=12.0 K 2024-11-24T22:15:44,143 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bd6592c37bb2:0-1 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/94f3ccf9f8a638fa974e1f69627832ef/.tmp/C/4920a0a11aed4fdfbbaa49f3afcc84b1 as hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/94f3ccf9f8a638fa974e1f69627832ef/C/4920a0a11aed4fdfbbaa49f3afcc84b1 2024-11-24T22:15:44,146 INFO [RS_FLUSH_OPERATIONS-regionserver/bd6592c37bb2:0-1 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/94f3ccf9f8a638fa974e1f69627832ef/C/4920a0a11aed4fdfbbaa49f3afcc84b1, entries=150, sequenceid=278, filesize=12.0 K 2024-11-24T22:15:44,147 INFO [RS_FLUSH_OPERATIONS-regionserver/bd6592c37bb2:0-1 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.HRegion(3140): Finished flush of dataSize ~107.34 KB/109920, heapSize ~319.45 KB/327120, currentSize=67.09 KB/68700 for 94f3ccf9f8a638fa974e1f69627832ef in 862ms, sequenceid=278, compaction requested=true 2024-11-24T22:15:44,147 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bd6592c37bb2:0-1 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.HRegion(2603): Flush status journal for 94f3ccf9f8a638fa974e1f69627832ef: 2024-11-24T22:15:44,147 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bd6592c37bb2:0-1 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732486521610.94f3ccf9f8a638fa974e1f69627832ef. 2024-11-24T22:15:44,147 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bd6592c37bb2:0-1 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=110 2024-11-24T22:15:44,148 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37701 {}] master.HMaster(4169): Remote procedure done, pid=110 2024-11-24T22:15:44,154 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=110, resume processing ppid=109 2024-11-24T22:15:44,155 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=110, ppid=109, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.0170 sec 2024-11-24T22:15:44,156 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=109, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=109, table=TestAcidGuarantees in 1.0240 sec 2024-11-24T22:15:44,271 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37701 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=109 2024-11-24T22:15:44,271 INFO [RPCClient-NioEventLoopGroup-4-8 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-24T22:15:44,272 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37701 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-24T22:15:44,273 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37701 {}] procedure2.ProcedureExecutor(1139): Stored pid=111, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=111, table=TestAcidGuarantees 2024-11-24T22:15:44,274 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37701 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=111 2024-11-24T22:15:44,274 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=111, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=111, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-24T22:15:44,275 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=111, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=111, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-24T22:15:44,275 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=112, ppid=111, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-24T22:15:44,381 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37701 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=111 2024-11-24T22:15:44,427 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=40087 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=112 2024-11-24T22:15:44,427 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bd6592c37bb2:0-2 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732486521610.94f3ccf9f8a638fa974e1f69627832ef. 2024-11-24T22:15:44,427 INFO [RS_FLUSH_OPERATIONS-regionserver/bd6592c37bb2:0-2 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.HRegion(2902): Flushing 94f3ccf9f8a638fa974e1f69627832ef 3/3 column families, dataSize=67.09 KB heapSize=199.97 KB 2024-11-24T22:15:44,427 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bd6592c37bb2:0-2 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 94f3ccf9f8a638fa974e1f69627832ef, store=A 2024-11-24T22:15:44,427 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bd6592c37bb2:0-2 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-24T22:15:44,427 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bd6592c37bb2:0-2 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 94f3ccf9f8a638fa974e1f69627832ef, store=B 2024-11-24T22:15:44,427 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bd6592c37bb2:0-2 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-24T22:15:44,427 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bd6592c37bb2:0-2 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 94f3ccf9f8a638fa974e1f69627832ef, store=C 2024-11-24T22:15:44,427 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bd6592c37bb2:0-2 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-24T22:15:44,433 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bd6592c37bb2:0-2 {event_type=RS_FLUSH_REGIONS, pid=112}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241124be89bcf0c9464cd7a6332888a52d26a0_94f3ccf9f8a638fa974e1f69627832ef is 50, key is test_row_0/A:col10/1732486543306/Put/seqid=0 2024-11-24T22:15:44,437 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45929 is added to blk_1073742260_1436 (size=12454) 2024-11-24T22:15:44,539 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40087 {}] regionserver.HRegion(8855): Flush requested on 94f3ccf9f8a638fa974e1f69627832ef 2024-11-24T22:15:44,539 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732486521610.94f3ccf9f8a638fa974e1f69627832ef. as already flushing 2024-11-24T22:15:44,591 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37701 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=111 2024-11-24T22:15:44,604 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40087 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94f3ccf9f8a638fa974e1f69627832ef, server=bd6592c37bb2,40087,1732486423713 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-24T22:15:44,604 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40087 {}] ipc.CallRunner(138): callId: 114 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:32846 deadline: 1732486604591, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94f3ccf9f8a638fa974e1f69627832ef, server=bd6592c37bb2,40087,1732486423713 2024-11-24T22:15:44,604 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732486521610.94f3ccf9f8a638fa974e1f69627832ef., hostname=bd6592c37bb2,40087,1732486423713, seqNum=5 , the old value is region=TestAcidGuarantees,,1732486521610.94f3ccf9f8a638fa974e1f69627832ef., hostname=bd6592c37bb2,40087,1732486423713, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94f3ccf9f8a638fa974e1f69627832ef, server=bd6592c37bb2,40087,1732486423713 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T22:15:44,604 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732486521610.94f3ccf9f8a638fa974e1f69627832ef., hostname=bd6592c37bb2,40087,1732486423713, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94f3ccf9f8a638fa974e1f69627832ef, server=bd6592c37bb2,40087,1732486423713 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T22:15:44,604 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732486521610.94f3ccf9f8a638fa974e1f69627832ef., hostname=bd6592c37bb2,40087,1732486423713, seqNum=5 because the exception is null or not the one we care about 2024-11-24T22:15:44,613 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40087 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94f3ccf9f8a638fa974e1f69627832ef, server=bd6592c37bb2,40087,1732486423713 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-24T22:15:44,613 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40087 {}] ipc.CallRunner(138): callId: 115 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:32774 deadline: 1732486604592, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94f3ccf9f8a638fa974e1f69627832ef, server=bd6592c37bb2,40087,1732486423713 2024-11-24T22:15:44,613 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732486521610.94f3ccf9f8a638fa974e1f69627832ef., hostname=bd6592c37bb2,40087,1732486423713, seqNum=5 , the old value is region=TestAcidGuarantees,,1732486521610.94f3ccf9f8a638fa974e1f69627832ef., hostname=bd6592c37bb2,40087,1732486423713, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94f3ccf9f8a638fa974e1f69627832ef, server=bd6592c37bb2,40087,1732486423713 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T22:15:44,613 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40087 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94f3ccf9f8a638fa974e1f69627832ef, server=bd6592c37bb2,40087,1732486423713 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-24T22:15:44,613 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732486521610.94f3ccf9f8a638fa974e1f69627832ef., hostname=bd6592c37bb2,40087,1732486423713, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94f3ccf9f8a638fa974e1f69627832ef, server=bd6592c37bb2,40087,1732486423713 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T22:15:44,613 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732486521610.94f3ccf9f8a638fa974e1f69627832ef., hostname=bd6592c37bb2,40087,1732486423713, seqNum=5 because the exception is null or not the one we care about 2024-11-24T22:15:44,613 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40087 {}] ipc.CallRunner(138): callId: 98 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:32832 deadline: 1732486604598, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94f3ccf9f8a638fa974e1f69627832ef, server=bd6592c37bb2,40087,1732486423713 2024-11-24T22:15:44,613 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732486521610.94f3ccf9f8a638fa974e1f69627832ef., hostname=bd6592c37bb2,40087,1732486423713, seqNum=5 , the old value is region=TestAcidGuarantees,,1732486521610.94f3ccf9f8a638fa974e1f69627832ef., hostname=bd6592c37bb2,40087,1732486423713, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94f3ccf9f8a638fa974e1f69627832ef, server=bd6592c37bb2,40087,1732486423713 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T22:15:44,613 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732486521610.94f3ccf9f8a638fa974e1f69627832ef., hostname=bd6592c37bb2,40087,1732486423713, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94f3ccf9f8a638fa974e1f69627832ef, server=bd6592c37bb2,40087,1732486423713 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T22:15:44,613 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732486521610.94f3ccf9f8a638fa974e1f69627832ef., hostname=bd6592c37bb2,40087,1732486423713, seqNum=5 because the exception is null or not the one we care about 2024-11-24T22:15:44,711 INFO [Thread-1608 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1608. 2024-11-24T22:15:44,711 DEBUG [Thread-1608 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicScanReader.workDone(AcidGuaranteesTestTool.java:302) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-24T22:15:44,711 DEBUG [Thread-1608 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-24T22:15:44,711 DEBUG [Thread-1608 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-24T22:15:44,711 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-24T22:15:44,716 INFO [Thread-1610 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1610. 2024-11-24T22:15:44,716 DEBUG [Thread-1610 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicScanReader.workDone(AcidGuaranteesTestTool.java:302) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-24T22:15:44,716 DEBUG [Thread-1610 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-24T22:15:44,716 DEBUG [Thread-1610 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-24T22:15:44,716 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-24T22:15:44,718 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40087 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94f3ccf9f8a638fa974e1f69627832ef, server=bd6592c37bb2,40087,1732486423713 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-24T22:15:44,718 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40087 {}] ipc.CallRunner(138): callId: 115 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:32846 deadline: 1732486604711, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94f3ccf9f8a638fa974e1f69627832ef, server=bd6592c37bb2,40087,1732486423713 2024-11-24T22:15:44,719 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732486521610.94f3ccf9f8a638fa974e1f69627832ef., hostname=bd6592c37bb2,40087,1732486423713, seqNum=5 , the old value is region=TestAcidGuarantees,,1732486521610.94f3ccf9f8a638fa974e1f69627832ef., hostname=bd6592c37bb2,40087,1732486423713, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94f3ccf9f8a638fa974e1f69627832ef, server=bd6592c37bb2,40087,1732486423713 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T22:15:44,719 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732486521610.94f3ccf9f8a638fa974e1f69627832ef., hostname=bd6592c37bb2,40087,1732486423713, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94f3ccf9f8a638fa974e1f69627832ef, server=bd6592c37bb2,40087,1732486423713 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T22:15:44,719 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732486521610.94f3ccf9f8a638fa974e1f69627832ef., hostname=bd6592c37bb2,40087,1732486423713, seqNum=5 because the exception is null or not the one we care about 2024-11-24T22:15:44,720 INFO [Thread-1606 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1606. 2024-11-24T22:15:44,720 DEBUG [Thread-1606 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicScanReader.workDone(AcidGuaranteesTestTool.java:302) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-24T22:15:44,720 DEBUG [Thread-1606 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-24T22:15:44,720 DEBUG [Thread-1606 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-24T22:15:44,720 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-24T22:15:44,721 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40087 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94f3ccf9f8a638fa974e1f69627832ef, server=bd6592c37bb2,40087,1732486423713 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-24T22:15:44,721 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40087 {}] ipc.CallRunner(138): callId: 116 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:32774 deadline: 1732486604721, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94f3ccf9f8a638fa974e1f69627832ef, server=bd6592c37bb2,40087,1732486423713 2024-11-24T22:15:44,722 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40087 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94f3ccf9f8a638fa974e1f69627832ef, server=bd6592c37bb2,40087,1732486423713 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-24T22:15:44,722 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732486521610.94f3ccf9f8a638fa974e1f69627832ef., hostname=bd6592c37bb2,40087,1732486423713, seqNum=5 , the old value is region=TestAcidGuarantees,,1732486521610.94f3ccf9f8a638fa974e1f69627832ef., hostname=bd6592c37bb2,40087,1732486423713, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94f3ccf9f8a638fa974e1f69627832ef, server=bd6592c37bb2,40087,1732486423713 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T22:15:44,722 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40087 {}] ipc.CallRunner(138): callId: 99 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:32832 deadline: 1732486604721, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94f3ccf9f8a638fa974e1f69627832ef, server=bd6592c37bb2,40087,1732486423713 2024-11-24T22:15:44,722 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732486521610.94f3ccf9f8a638fa974e1f69627832ef., hostname=bd6592c37bb2,40087,1732486423713, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94f3ccf9f8a638fa974e1f69627832ef, server=bd6592c37bb2,40087,1732486423713 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T22:15:44,722 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732486521610.94f3ccf9f8a638fa974e1f69627832ef., hostname=bd6592c37bb2,40087,1732486423713, seqNum=5 because the exception is null or not the one we care about 2024-11-24T22:15:44,722 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732486521610.94f3ccf9f8a638fa974e1f69627832ef., hostname=bd6592c37bb2,40087,1732486423713, seqNum=5 , the old value is region=TestAcidGuarantees,,1732486521610.94f3ccf9f8a638fa974e1f69627832ef., hostname=bd6592c37bb2,40087,1732486423713, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94f3ccf9f8a638fa974e1f69627832ef, server=bd6592c37bb2,40087,1732486423713 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T22:15:44,722 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732486521610.94f3ccf9f8a638fa974e1f69627832ef., hostname=bd6592c37bb2,40087,1732486423713, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94f3ccf9f8a638fa974e1f69627832ef, server=bd6592c37bb2,40087,1732486423713 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T22:15:44,722 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732486521610.94f3ccf9f8a638fa974e1f69627832ef., hostname=bd6592c37bb2,40087,1732486423713, seqNum=5 because the exception is null or not the one we care about 2024-11-24T22:15:44,726 INFO [Thread-1612 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1612. 2024-11-24T22:15:44,726 DEBUG [Thread-1612 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicScanReader.workDone(AcidGuaranteesTestTool.java:302) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-24T22:15:44,726 DEBUG [Thread-1612 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-24T22:15:44,726 DEBUG [Thread-1612 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-24T22:15:44,726 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-24T22:15:44,730 INFO [Thread-1614 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1614. 2024-11-24T22:15:44,730 DEBUG [Thread-1614 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicScanReader.workDone(AcidGuaranteesTestTool.java:302) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-24T22:15:44,730 DEBUG [Thread-1614 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-24T22:15:44,730 DEBUG [Thread-1614 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-24T22:15:44,730 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-24T22:15:44,837 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bd6592c37bb2:0-2 {event_type=RS_FLUSH_REGIONS, pid=112}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T22:15:44,840 INFO [RS_FLUSH_OPERATIONS-regionserver/bd6592c37bb2:0-2 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241124be89bcf0c9464cd7a6332888a52d26a0_94f3ccf9f8a638fa974e1f69627832ef to hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241124be89bcf0c9464cd7a6332888a52d26a0_94f3ccf9f8a638fa974e1f69627832ef 2024-11-24T22:15:44,841 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bd6592c37bb2:0-2 {event_type=RS_FLUSH_REGIONS, pid=112}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/94f3ccf9f8a638fa974e1f69627832ef/.tmp/A/510a774ece96416f8abbaee4f4c581cd, store: [table=TestAcidGuarantees family=A region=94f3ccf9f8a638fa974e1f69627832ef] 2024-11-24T22:15:44,841 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bd6592c37bb2:0-2 {event_type=RS_FLUSH_REGIONS, pid=112}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/94f3ccf9f8a638fa974e1f69627832ef/.tmp/A/510a774ece96416f8abbaee4f4c581cd is 175, key is test_row_0/A:col10/1732486543306/Put/seqid=0 2024-11-24T22:15:44,844 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45929 is added to blk_1073742261_1437 (size=31255) 2024-11-24T22:15:44,900 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37701 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=111 2024-11-24T22:15:44,931 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40087 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94f3ccf9f8a638fa974e1f69627832ef, server=bd6592c37bb2,40087,1732486423713 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-24T22:15:44,931 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40087 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94f3ccf9f8a638fa974e1f69627832ef, server=bd6592c37bb2,40087,1732486423713 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-24T22:15:44,931 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40087 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94f3ccf9f8a638fa974e1f69627832ef, server=bd6592c37bb2,40087,1732486423713 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-24T22:15:44,931 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40087 {}] ipc.CallRunner(138): callId: 116 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:32846 deadline: 1732486604931, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94f3ccf9f8a638fa974e1f69627832ef, server=bd6592c37bb2,40087,1732486423713 2024-11-24T22:15:44,931 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40087 {}] ipc.CallRunner(138): callId: 117 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:32774 deadline: 1732486604931, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94f3ccf9f8a638fa974e1f69627832ef, server=bd6592c37bb2,40087,1732486423713 2024-11-24T22:15:44,931 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40087 {}] ipc.CallRunner(138): callId: 100 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:32832 deadline: 1732486604931, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94f3ccf9f8a638fa974e1f69627832ef, server=bd6592c37bb2,40087,1732486423713 2024-11-24T22:15:44,931 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732486521610.94f3ccf9f8a638fa974e1f69627832ef., hostname=bd6592c37bb2,40087,1732486423713, seqNum=5 , the old value is region=TestAcidGuarantees,,1732486521610.94f3ccf9f8a638fa974e1f69627832ef., hostname=bd6592c37bb2,40087,1732486423713, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94f3ccf9f8a638fa974e1f69627832ef, server=bd6592c37bb2,40087,1732486423713 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T22:15:44,931 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732486521610.94f3ccf9f8a638fa974e1f69627832ef., hostname=bd6592c37bb2,40087,1732486423713, seqNum=5 , the old value is region=TestAcidGuarantees,,1732486521610.94f3ccf9f8a638fa974e1f69627832ef., hostname=bd6592c37bb2,40087,1732486423713, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94f3ccf9f8a638fa974e1f69627832ef, server=bd6592c37bb2,40087,1732486423713 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T22:15:44,931 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732486521610.94f3ccf9f8a638fa974e1f69627832ef., hostname=bd6592c37bb2,40087,1732486423713, seqNum=5 , the old value is region=TestAcidGuarantees,,1732486521610.94f3ccf9f8a638fa974e1f69627832ef., hostname=bd6592c37bb2,40087,1732486423713, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94f3ccf9f8a638fa974e1f69627832ef, server=bd6592c37bb2,40087,1732486423713 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T22:15:44,931 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732486521610.94f3ccf9f8a638fa974e1f69627832ef., hostname=bd6592c37bb2,40087,1732486423713, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94f3ccf9f8a638fa974e1f69627832ef, server=bd6592c37bb2,40087,1732486423713 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T22:15:44,931 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732486521610.94f3ccf9f8a638fa974e1f69627832ef., hostname=bd6592c37bb2,40087,1732486423713, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94f3ccf9f8a638fa974e1f69627832ef, server=bd6592c37bb2,40087,1732486423713 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T22:15:44,932 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732486521610.94f3ccf9f8a638fa974e1f69627832ef., hostname=bd6592c37bb2,40087,1732486423713, seqNum=5 because the exception is null or not the one we care about 2024-11-24T22:15:44,932 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732486521610.94f3ccf9f8a638fa974e1f69627832ef., hostname=bd6592c37bb2,40087,1732486423713, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94f3ccf9f8a638fa974e1f69627832ef, server=bd6592c37bb2,40087,1732486423713 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T22:15:44,932 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732486521610.94f3ccf9f8a638fa974e1f69627832ef., hostname=bd6592c37bb2,40087,1732486423713, seqNum=5 because the exception is null or not the one we care about 2024-11-24T22:15:44,932 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732486521610.94f3ccf9f8a638fa974e1f69627832ef., hostname=bd6592c37bb2,40087,1732486423713, seqNum=5 because the exception is null or not the one we care about 2024-11-24T22:15:45,241 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40087 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94f3ccf9f8a638fa974e1f69627832ef, server=bd6592c37bb2,40087,1732486423713 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-24T22:15:45,241 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40087 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94f3ccf9f8a638fa974e1f69627832ef, server=bd6592c37bb2,40087,1732486423713 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-24T22:15:45,241 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40087 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94f3ccf9f8a638fa974e1f69627832ef, server=bd6592c37bb2,40087,1732486423713 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-24T22:15:45,242 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40087 {}] ipc.CallRunner(138): callId: 117 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:32846 deadline: 1732486605241, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94f3ccf9f8a638fa974e1f69627832ef, server=bd6592c37bb2,40087,1732486423713 2024-11-24T22:15:45,242 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40087 {}] ipc.CallRunner(138): callId: 118 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:32774 deadline: 1732486605241, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94f3ccf9f8a638fa974e1f69627832ef, server=bd6592c37bb2,40087,1732486423713 2024-11-24T22:15:45,242 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40087 {}] ipc.CallRunner(138): callId: 101 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:32832 deadline: 1732486605241, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94f3ccf9f8a638fa974e1f69627832ef, server=bd6592c37bb2,40087,1732486423713 2024-11-24T22:15:45,242 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732486521610.94f3ccf9f8a638fa974e1f69627832ef., hostname=bd6592c37bb2,40087,1732486423713, seqNum=5 , the old value is region=TestAcidGuarantees,,1732486521610.94f3ccf9f8a638fa974e1f69627832ef., hostname=bd6592c37bb2,40087,1732486423713, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94f3ccf9f8a638fa974e1f69627832ef, server=bd6592c37bb2,40087,1732486423713 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T22:15:45,242 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732486521610.94f3ccf9f8a638fa974e1f69627832ef., hostname=bd6592c37bb2,40087,1732486423713, seqNum=5 , the old value is region=TestAcidGuarantees,,1732486521610.94f3ccf9f8a638fa974e1f69627832ef., hostname=bd6592c37bb2,40087,1732486423713, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94f3ccf9f8a638fa974e1f69627832ef, server=bd6592c37bb2,40087,1732486423713 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T22:15:45,242 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732486521610.94f3ccf9f8a638fa974e1f69627832ef., hostname=bd6592c37bb2,40087,1732486423713, seqNum=5 , the old value is region=TestAcidGuarantees,,1732486521610.94f3ccf9f8a638fa974e1f69627832ef., hostname=bd6592c37bb2,40087,1732486423713, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94f3ccf9f8a638fa974e1f69627832ef, server=bd6592c37bb2,40087,1732486423713 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T22:15:45,242 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732486521610.94f3ccf9f8a638fa974e1f69627832ef., hostname=bd6592c37bb2,40087,1732486423713, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94f3ccf9f8a638fa974e1f69627832ef, server=bd6592c37bb2,40087,1732486423713 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T22:15:45,242 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732486521610.94f3ccf9f8a638fa974e1f69627832ef., hostname=bd6592c37bb2,40087,1732486423713, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94f3ccf9f8a638fa974e1f69627832ef, server=bd6592c37bb2,40087,1732486423713 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T22:15:45,242 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732486521610.94f3ccf9f8a638fa974e1f69627832ef., hostname=bd6592c37bb2,40087,1732486423713, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94f3ccf9f8a638fa974e1f69627832ef, server=bd6592c37bb2,40087,1732486423713 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T22:15:45,242 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732486521610.94f3ccf9f8a638fa974e1f69627832ef., hostname=bd6592c37bb2,40087,1732486423713, seqNum=5 because the exception is null or not the one we care about 2024-11-24T22:15:45,242 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732486521610.94f3ccf9f8a638fa974e1f69627832ef., hostname=bd6592c37bb2,40087,1732486423713, seqNum=5 because the exception is null or not the one we care about 2024-11-24T22:15:45,242 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732486521610.94f3ccf9f8a638fa974e1f69627832ef., hostname=bd6592c37bb2,40087,1732486423713, seqNum=5 because the exception is null or not the one we care about 2024-11-24T22:15:45,245 INFO [RS_FLUSH_OPERATIONS-regionserver/bd6592c37bb2:0-2 {event_type=RS_FLUSH_REGIONS, pid=112}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=291, memsize=22.4 K, hasBloomFilter=true, into tmp file hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/94f3ccf9f8a638fa974e1f69627832ef/.tmp/A/510a774ece96416f8abbaee4f4c581cd 2024-11-24T22:15:45,250 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bd6592c37bb2:0-2 {event_type=RS_FLUSH_REGIONS, pid=112}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/94f3ccf9f8a638fa974e1f69627832ef/.tmp/B/32aece1bb9144ffb8c84c5135450995b is 50, key is test_row_0/B:col10/1732486543306/Put/seqid=0 2024-11-24T22:15:45,253 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45929 is added to blk_1073742262_1438 (size=12301) 2024-11-24T22:15:45,375 DEBUG [regionserver/bd6592c37bb2:0.Chore.1 {}] throttle.PressureAwareCompactionThroughputController(103): CompactionPressure is 0.15384615384615385, tune throughput to 57.69 MB/second 2024-11-24T22:15:45,377 WARN [regionserver/bd6592c37bb2:0.Chore.1 {}] hbase.ExecutorStatusChore(69): RS_COMPACTED_FILES_DISCHARGER's size info, queued: 6, running: 1 2024-11-24T22:15:45,377 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/bd6592c37bb2:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] regionserver.HStore(2317): Moving the files [hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/94f3ccf9f8a638fa974e1f69627832ef/A/a8661fe5f09945f8996df4fc0affd5f9, hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/94f3ccf9f8a638fa974e1f69627832ef/A/eea8340353634d808eee2f348943af0e, hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/94f3ccf9f8a638fa974e1f69627832ef/A/8ce2903e8a5b46edb21934de4a278e2d, hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/94f3ccf9f8a638fa974e1f69627832ef/A/1b69565af4094bd79b4d460cb7671894, hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/94f3ccf9f8a638fa974e1f69627832ef/A/95ed96d635f54bce8685e4aa528975e7, hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/94f3ccf9f8a638fa974e1f69627832ef/A/89ef506c9e084dff96cbbd975d3a86cb, hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/94f3ccf9f8a638fa974e1f69627832ef/A/83f546c8ac3a4c0dbfe435b59d1e9387, hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/94f3ccf9f8a638fa974e1f69627832ef/A/e78a5bea000d4b5088fbdfd1f6b3d2f8, hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/94f3ccf9f8a638fa974e1f69627832ef/A/3e23a020d2d641a09079ca5e307d947e, hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/94f3ccf9f8a638fa974e1f69627832ef/A/fce104982b484448966f2a223c8fc392, hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/94f3ccf9f8a638fa974e1f69627832ef/A/697a1699f64d45fcb65780147612779b, hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/94f3ccf9f8a638fa974e1f69627832ef/A/c3b3073483d94256bdd7bddae194779a, hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/94f3ccf9f8a638fa974e1f69627832ef/A/e25b282932044a15acb1a40c17957984, hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/94f3ccf9f8a638fa974e1f69627832ef/A/17c6ecafbb0d4ff6b176abbe1aed6b68, hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/94f3ccf9f8a638fa974e1f69627832ef/A/28d65c9086ac4081b22ae0ce11933e15, hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/94f3ccf9f8a638fa974e1f69627832ef/A/546c086e2e0c4208ba4af9d5985b3e9b] to archive 2024-11-24T22:15:45,378 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/bd6592c37bb2:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(360): Archiving compacted files. 2024-11-24T22:15:45,379 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/bd6592c37bb2:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/94f3ccf9f8a638fa974e1f69627832ef/A/a8661fe5f09945f8996df4fc0affd5f9 to hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/archive/data/default/TestAcidGuarantees/94f3ccf9f8a638fa974e1f69627832ef/A/a8661fe5f09945f8996df4fc0affd5f9 2024-11-24T22:15:45,380 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/bd6592c37bb2:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/94f3ccf9f8a638fa974e1f69627832ef/A/eea8340353634d808eee2f348943af0e to hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/archive/data/default/TestAcidGuarantees/94f3ccf9f8a638fa974e1f69627832ef/A/eea8340353634d808eee2f348943af0e 2024-11-24T22:15:45,381 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/bd6592c37bb2:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/94f3ccf9f8a638fa974e1f69627832ef/A/8ce2903e8a5b46edb21934de4a278e2d to hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/archive/data/default/TestAcidGuarantees/94f3ccf9f8a638fa974e1f69627832ef/A/8ce2903e8a5b46edb21934de4a278e2d 2024-11-24T22:15:45,382 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/bd6592c37bb2:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/94f3ccf9f8a638fa974e1f69627832ef/A/1b69565af4094bd79b4d460cb7671894 to hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/archive/data/default/TestAcidGuarantees/94f3ccf9f8a638fa974e1f69627832ef/A/1b69565af4094bd79b4d460cb7671894 2024-11-24T22:15:45,383 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/bd6592c37bb2:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/94f3ccf9f8a638fa974e1f69627832ef/A/95ed96d635f54bce8685e4aa528975e7 to hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/archive/data/default/TestAcidGuarantees/94f3ccf9f8a638fa974e1f69627832ef/A/95ed96d635f54bce8685e4aa528975e7 2024-11-24T22:15:45,383 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/bd6592c37bb2:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/94f3ccf9f8a638fa974e1f69627832ef/A/89ef506c9e084dff96cbbd975d3a86cb to hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/archive/data/default/TestAcidGuarantees/94f3ccf9f8a638fa974e1f69627832ef/A/89ef506c9e084dff96cbbd975d3a86cb 2024-11-24T22:15:45,384 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/bd6592c37bb2:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/94f3ccf9f8a638fa974e1f69627832ef/A/83f546c8ac3a4c0dbfe435b59d1e9387 to hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/archive/data/default/TestAcidGuarantees/94f3ccf9f8a638fa974e1f69627832ef/A/83f546c8ac3a4c0dbfe435b59d1e9387 2024-11-24T22:15:45,385 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/bd6592c37bb2:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/94f3ccf9f8a638fa974e1f69627832ef/A/e78a5bea000d4b5088fbdfd1f6b3d2f8 to hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/archive/data/default/TestAcidGuarantees/94f3ccf9f8a638fa974e1f69627832ef/A/e78a5bea000d4b5088fbdfd1f6b3d2f8 2024-11-24T22:15:45,386 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/bd6592c37bb2:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/94f3ccf9f8a638fa974e1f69627832ef/A/3e23a020d2d641a09079ca5e307d947e to hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/archive/data/default/TestAcidGuarantees/94f3ccf9f8a638fa974e1f69627832ef/A/3e23a020d2d641a09079ca5e307d947e 2024-11-24T22:15:45,386 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/bd6592c37bb2:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/94f3ccf9f8a638fa974e1f69627832ef/A/fce104982b484448966f2a223c8fc392 to hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/archive/data/default/TestAcidGuarantees/94f3ccf9f8a638fa974e1f69627832ef/A/fce104982b484448966f2a223c8fc392 2024-11-24T22:15:45,387 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/bd6592c37bb2:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/94f3ccf9f8a638fa974e1f69627832ef/A/697a1699f64d45fcb65780147612779b to hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/archive/data/default/TestAcidGuarantees/94f3ccf9f8a638fa974e1f69627832ef/A/697a1699f64d45fcb65780147612779b 2024-11-24T22:15:45,388 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/bd6592c37bb2:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/94f3ccf9f8a638fa974e1f69627832ef/A/c3b3073483d94256bdd7bddae194779a to hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/archive/data/default/TestAcidGuarantees/94f3ccf9f8a638fa974e1f69627832ef/A/c3b3073483d94256bdd7bddae194779a 2024-11-24T22:15:45,389 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/bd6592c37bb2:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/94f3ccf9f8a638fa974e1f69627832ef/A/e25b282932044a15acb1a40c17957984 to hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/archive/data/default/TestAcidGuarantees/94f3ccf9f8a638fa974e1f69627832ef/A/e25b282932044a15acb1a40c17957984 2024-11-24T22:15:45,390 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/bd6592c37bb2:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/94f3ccf9f8a638fa974e1f69627832ef/A/17c6ecafbb0d4ff6b176abbe1aed6b68 to hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/archive/data/default/TestAcidGuarantees/94f3ccf9f8a638fa974e1f69627832ef/A/17c6ecafbb0d4ff6b176abbe1aed6b68 2024-11-24T22:15:45,391 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/bd6592c37bb2:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/94f3ccf9f8a638fa974e1f69627832ef/A/28d65c9086ac4081b22ae0ce11933e15 to hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/archive/data/default/TestAcidGuarantees/94f3ccf9f8a638fa974e1f69627832ef/A/28d65c9086ac4081b22ae0ce11933e15 2024-11-24T22:15:45,391 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/bd6592c37bb2:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/94f3ccf9f8a638fa974e1f69627832ef/A/546c086e2e0c4208ba4af9d5985b3e9b to hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/archive/data/default/TestAcidGuarantees/94f3ccf9f8a638fa974e1f69627832ef/A/546c086e2e0c4208ba4af9d5985b3e9b 2024-11-24T22:15:45,393 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/bd6592c37bb2:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] regionserver.HStore(2317): Moving the files [hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/94f3ccf9f8a638fa974e1f69627832ef/B/cd94decc27924b9687ddf2ec811ae351, hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/94f3ccf9f8a638fa974e1f69627832ef/B/cc5afda838d84551b7b988a9ea3631f6, hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/94f3ccf9f8a638fa974e1f69627832ef/B/e6bb3a30f1f848dd9dea06828499b452, hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/94f3ccf9f8a638fa974e1f69627832ef/B/178dfc664e9c455d93e59f22305934cf, hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/94f3ccf9f8a638fa974e1f69627832ef/B/26545f645a554d1daddca6dbf2adc469, hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/94f3ccf9f8a638fa974e1f69627832ef/B/d8beb157656c45efade39f49eb755238, hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/94f3ccf9f8a638fa974e1f69627832ef/B/f12630c898be400d87e378aa2a6ebac3, hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/94f3ccf9f8a638fa974e1f69627832ef/B/97bd3de9d98b49909095a4474b63e142, hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/94f3ccf9f8a638fa974e1f69627832ef/B/77b19474d2d548f7916c60b8c12adad0, hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/94f3ccf9f8a638fa974e1f69627832ef/B/1c3c63e6b7a943afbb4d603545e4dd0a, hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/94f3ccf9f8a638fa974e1f69627832ef/B/d141ed51dbd0448f9084a2a5552ef064, hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/94f3ccf9f8a638fa974e1f69627832ef/B/aed5d43a422c4c0a892806e5dc6d9ddb, hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/94f3ccf9f8a638fa974e1f69627832ef/B/bb9965a6ef4f4109a98514be227d61cf, hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/94f3ccf9f8a638fa974e1f69627832ef/B/853e51745e104112b072f41f1a4b6852, hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/94f3ccf9f8a638fa974e1f69627832ef/B/72127362fbeb4a0598f8132387bbe316, hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/94f3ccf9f8a638fa974e1f69627832ef/B/39ded812f39945be83f4845513cfe2f2] to archive 2024-11-24T22:15:45,394 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/bd6592c37bb2:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(360): Archiving compacted files. 2024-11-24T22:15:45,395 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/bd6592c37bb2:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/94f3ccf9f8a638fa974e1f69627832ef/B/cd94decc27924b9687ddf2ec811ae351 to hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/archive/data/default/TestAcidGuarantees/94f3ccf9f8a638fa974e1f69627832ef/B/cd94decc27924b9687ddf2ec811ae351 2024-11-24T22:15:45,396 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/bd6592c37bb2:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/94f3ccf9f8a638fa974e1f69627832ef/B/cc5afda838d84551b7b988a9ea3631f6 to hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/archive/data/default/TestAcidGuarantees/94f3ccf9f8a638fa974e1f69627832ef/B/cc5afda838d84551b7b988a9ea3631f6 2024-11-24T22:15:45,397 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/bd6592c37bb2:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/94f3ccf9f8a638fa974e1f69627832ef/B/e6bb3a30f1f848dd9dea06828499b452 to hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/archive/data/default/TestAcidGuarantees/94f3ccf9f8a638fa974e1f69627832ef/B/e6bb3a30f1f848dd9dea06828499b452 2024-11-24T22:15:45,397 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/bd6592c37bb2:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/94f3ccf9f8a638fa974e1f69627832ef/B/178dfc664e9c455d93e59f22305934cf to hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/archive/data/default/TestAcidGuarantees/94f3ccf9f8a638fa974e1f69627832ef/B/178dfc664e9c455d93e59f22305934cf 2024-11-24T22:15:45,398 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/bd6592c37bb2:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/94f3ccf9f8a638fa974e1f69627832ef/B/26545f645a554d1daddca6dbf2adc469 to hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/archive/data/default/TestAcidGuarantees/94f3ccf9f8a638fa974e1f69627832ef/B/26545f645a554d1daddca6dbf2adc469 2024-11-24T22:15:45,399 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/bd6592c37bb2:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/94f3ccf9f8a638fa974e1f69627832ef/B/d8beb157656c45efade39f49eb755238 to hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/archive/data/default/TestAcidGuarantees/94f3ccf9f8a638fa974e1f69627832ef/B/d8beb157656c45efade39f49eb755238 2024-11-24T22:15:45,399 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/bd6592c37bb2:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/94f3ccf9f8a638fa974e1f69627832ef/B/f12630c898be400d87e378aa2a6ebac3 to hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/archive/data/default/TestAcidGuarantees/94f3ccf9f8a638fa974e1f69627832ef/B/f12630c898be400d87e378aa2a6ebac3 2024-11-24T22:15:45,400 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/bd6592c37bb2:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/94f3ccf9f8a638fa974e1f69627832ef/B/97bd3de9d98b49909095a4474b63e142 to hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/archive/data/default/TestAcidGuarantees/94f3ccf9f8a638fa974e1f69627832ef/B/97bd3de9d98b49909095a4474b63e142 2024-11-24T22:15:45,401 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/bd6592c37bb2:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/94f3ccf9f8a638fa974e1f69627832ef/B/77b19474d2d548f7916c60b8c12adad0 to hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/archive/data/default/TestAcidGuarantees/94f3ccf9f8a638fa974e1f69627832ef/B/77b19474d2d548f7916c60b8c12adad0 2024-11-24T22:15:45,401 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/bd6592c37bb2:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/94f3ccf9f8a638fa974e1f69627832ef/B/1c3c63e6b7a943afbb4d603545e4dd0a to hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/archive/data/default/TestAcidGuarantees/94f3ccf9f8a638fa974e1f69627832ef/B/1c3c63e6b7a943afbb4d603545e4dd0a 2024-11-24T22:15:45,402 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/bd6592c37bb2:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/94f3ccf9f8a638fa974e1f69627832ef/B/d141ed51dbd0448f9084a2a5552ef064 to hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/archive/data/default/TestAcidGuarantees/94f3ccf9f8a638fa974e1f69627832ef/B/d141ed51dbd0448f9084a2a5552ef064 2024-11-24T22:15:45,402 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/bd6592c37bb2:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/94f3ccf9f8a638fa974e1f69627832ef/B/aed5d43a422c4c0a892806e5dc6d9ddb to hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/archive/data/default/TestAcidGuarantees/94f3ccf9f8a638fa974e1f69627832ef/B/aed5d43a422c4c0a892806e5dc6d9ddb 2024-11-24T22:15:45,403 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/bd6592c37bb2:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/94f3ccf9f8a638fa974e1f69627832ef/B/bb9965a6ef4f4109a98514be227d61cf to hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/archive/data/default/TestAcidGuarantees/94f3ccf9f8a638fa974e1f69627832ef/B/bb9965a6ef4f4109a98514be227d61cf 2024-11-24T22:15:45,404 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/bd6592c37bb2:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/94f3ccf9f8a638fa974e1f69627832ef/B/853e51745e104112b072f41f1a4b6852 to hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/archive/data/default/TestAcidGuarantees/94f3ccf9f8a638fa974e1f69627832ef/B/853e51745e104112b072f41f1a4b6852 2024-11-24T22:15:45,405 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/bd6592c37bb2:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/94f3ccf9f8a638fa974e1f69627832ef/B/72127362fbeb4a0598f8132387bbe316 to hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/archive/data/default/TestAcidGuarantees/94f3ccf9f8a638fa974e1f69627832ef/B/72127362fbeb4a0598f8132387bbe316 2024-11-24T22:15:45,405 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/bd6592c37bb2:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/94f3ccf9f8a638fa974e1f69627832ef/B/39ded812f39945be83f4845513cfe2f2 to hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/archive/data/default/TestAcidGuarantees/94f3ccf9f8a638fa974e1f69627832ef/B/39ded812f39945be83f4845513cfe2f2 2024-11-24T22:15:45,408 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/bd6592c37bb2:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] regionserver.HStore(2317): Moving the files [hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/94f3ccf9f8a638fa974e1f69627832ef/C/0c6077b470f14406ad8905d5895c98ee, hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/94f3ccf9f8a638fa974e1f69627832ef/C/a6f6fe973984491a9e1d7cecd110be45, hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/94f3ccf9f8a638fa974e1f69627832ef/C/2ba25fa3975e49de8bfb037efe930ba5, hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/94f3ccf9f8a638fa974e1f69627832ef/C/267a2d67eb91491c826a9d1801fe9d29, hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/94f3ccf9f8a638fa974e1f69627832ef/C/e003126d696c418db6a92bf6d9e16671, hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/94f3ccf9f8a638fa974e1f69627832ef/C/9b339451e3df4507bf90584aa37d34bd, hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/94f3ccf9f8a638fa974e1f69627832ef/C/f90e75ec41424220862d0a6950395ea9, hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/94f3ccf9f8a638fa974e1f69627832ef/C/cb3ac673639340e892a044d258cbd444, hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/94f3ccf9f8a638fa974e1f69627832ef/C/fc561c7bb18d48ffa58362322051fad8, hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/94f3ccf9f8a638fa974e1f69627832ef/C/a3395d0e4c1c43a98e2163a3d14415df, hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/94f3ccf9f8a638fa974e1f69627832ef/C/a12ca2c795164aae8da91298a1edd817, hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/94f3ccf9f8a638fa974e1f69627832ef/C/d067a887518d4c33a3df8668caf4c2da, hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/94f3ccf9f8a638fa974e1f69627832ef/C/c68109f6bef04e81a21e61cdb622d3f0, hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/94f3ccf9f8a638fa974e1f69627832ef/C/644e239d0c2a44909d1456f6cd2cb2b2, hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/94f3ccf9f8a638fa974e1f69627832ef/C/1f70e3bbb0e24ae8b788a7fc0abb2288] to archive 2024-11-24T22:15:45,409 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/bd6592c37bb2:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(360): Archiving compacted files. 2024-11-24T22:15:45,410 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/bd6592c37bb2:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/94f3ccf9f8a638fa974e1f69627832ef/C/0c6077b470f14406ad8905d5895c98ee to hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/archive/data/default/TestAcidGuarantees/94f3ccf9f8a638fa974e1f69627832ef/C/0c6077b470f14406ad8905d5895c98ee 2024-11-24T22:15:45,410 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37701 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=111 2024-11-24T22:15:45,411 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/bd6592c37bb2:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/94f3ccf9f8a638fa974e1f69627832ef/C/a6f6fe973984491a9e1d7cecd110be45 to hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/archive/data/default/TestAcidGuarantees/94f3ccf9f8a638fa974e1f69627832ef/C/a6f6fe973984491a9e1d7cecd110be45 2024-11-24T22:15:45,412 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/bd6592c37bb2:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/94f3ccf9f8a638fa974e1f69627832ef/C/2ba25fa3975e49de8bfb037efe930ba5 to hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/archive/data/default/TestAcidGuarantees/94f3ccf9f8a638fa974e1f69627832ef/C/2ba25fa3975e49de8bfb037efe930ba5 2024-11-24T22:15:45,413 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/bd6592c37bb2:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/94f3ccf9f8a638fa974e1f69627832ef/C/267a2d67eb91491c826a9d1801fe9d29 to hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/archive/data/default/TestAcidGuarantees/94f3ccf9f8a638fa974e1f69627832ef/C/267a2d67eb91491c826a9d1801fe9d29 2024-11-24T22:15:45,414 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/bd6592c37bb2:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/94f3ccf9f8a638fa974e1f69627832ef/C/e003126d696c418db6a92bf6d9e16671 to hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/archive/data/default/TestAcidGuarantees/94f3ccf9f8a638fa974e1f69627832ef/C/e003126d696c418db6a92bf6d9e16671 2024-11-24T22:15:45,415 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/bd6592c37bb2:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/94f3ccf9f8a638fa974e1f69627832ef/C/9b339451e3df4507bf90584aa37d34bd to hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/archive/data/default/TestAcidGuarantees/94f3ccf9f8a638fa974e1f69627832ef/C/9b339451e3df4507bf90584aa37d34bd 2024-11-24T22:15:45,416 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/bd6592c37bb2:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/94f3ccf9f8a638fa974e1f69627832ef/C/f90e75ec41424220862d0a6950395ea9 to hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/archive/data/default/TestAcidGuarantees/94f3ccf9f8a638fa974e1f69627832ef/C/f90e75ec41424220862d0a6950395ea9 2024-11-24T22:15:45,417 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/bd6592c37bb2:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/94f3ccf9f8a638fa974e1f69627832ef/C/cb3ac673639340e892a044d258cbd444 to hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/archive/data/default/TestAcidGuarantees/94f3ccf9f8a638fa974e1f69627832ef/C/cb3ac673639340e892a044d258cbd444 2024-11-24T22:15:45,418 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/bd6592c37bb2:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/94f3ccf9f8a638fa974e1f69627832ef/C/fc561c7bb18d48ffa58362322051fad8 to hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/archive/data/default/TestAcidGuarantees/94f3ccf9f8a638fa974e1f69627832ef/C/fc561c7bb18d48ffa58362322051fad8 2024-11-24T22:15:45,419 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/bd6592c37bb2:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/94f3ccf9f8a638fa974e1f69627832ef/C/a3395d0e4c1c43a98e2163a3d14415df to hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/archive/data/default/TestAcidGuarantees/94f3ccf9f8a638fa974e1f69627832ef/C/a3395d0e4c1c43a98e2163a3d14415df 2024-11-24T22:15:45,419 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/bd6592c37bb2:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/94f3ccf9f8a638fa974e1f69627832ef/C/a12ca2c795164aae8da91298a1edd817 to hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/archive/data/default/TestAcidGuarantees/94f3ccf9f8a638fa974e1f69627832ef/C/a12ca2c795164aae8da91298a1edd817 2024-11-24T22:15:45,420 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/bd6592c37bb2:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/94f3ccf9f8a638fa974e1f69627832ef/C/d067a887518d4c33a3df8668caf4c2da to hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/archive/data/default/TestAcidGuarantees/94f3ccf9f8a638fa974e1f69627832ef/C/d067a887518d4c33a3df8668caf4c2da 2024-11-24T22:15:45,421 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/bd6592c37bb2:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/94f3ccf9f8a638fa974e1f69627832ef/C/c68109f6bef04e81a21e61cdb622d3f0 to hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/archive/data/default/TestAcidGuarantees/94f3ccf9f8a638fa974e1f69627832ef/C/c68109f6bef04e81a21e61cdb622d3f0 2024-11-24T22:15:45,422 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/bd6592c37bb2:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/94f3ccf9f8a638fa974e1f69627832ef/C/644e239d0c2a44909d1456f6cd2cb2b2 to hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/archive/data/default/TestAcidGuarantees/94f3ccf9f8a638fa974e1f69627832ef/C/644e239d0c2a44909d1456f6cd2cb2b2 2024-11-24T22:15:45,423 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/bd6592c37bb2:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/94f3ccf9f8a638fa974e1f69627832ef/C/1f70e3bbb0e24ae8b788a7fc0abb2288 to hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/archive/data/default/TestAcidGuarantees/94f3ccf9f8a638fa974e1f69627832ef/C/1f70e3bbb0e24ae8b788a7fc0abb2288 2024-11-24T22:15:45,654 INFO [RS_FLUSH_OPERATIONS-regionserver/bd6592c37bb2:0-2 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=291 (bloomFilter=true), to=hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/94f3ccf9f8a638fa974e1f69627832ef/.tmp/B/32aece1bb9144ffb8c84c5135450995b 2024-11-24T22:15:45,659 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bd6592c37bb2:0-2 {event_type=RS_FLUSH_REGIONS, pid=112}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/94f3ccf9f8a638fa974e1f69627832ef/.tmp/C/d8e07a696b374dd38a0665746ee004b3 is 50, key is test_row_0/C:col10/1732486543306/Put/seqid=0 2024-11-24T22:15:45,662 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45929 is added to blk_1073742263_1439 (size=12301) 2024-11-24T22:15:45,751 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40087 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94f3ccf9f8a638fa974e1f69627832ef, server=bd6592c37bb2,40087,1732486423713 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-24T22:15:45,751 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40087 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94f3ccf9f8a638fa974e1f69627832ef, server=bd6592c37bb2,40087,1732486423713 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-24T22:15:45,751 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40087 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94f3ccf9f8a638fa974e1f69627832ef, server=bd6592c37bb2,40087,1732486423713 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-24T22:15:45,751 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40087 {}] ipc.CallRunner(138): callId: 119 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:32774 deadline: 1732486605751, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94f3ccf9f8a638fa974e1f69627832ef, server=bd6592c37bb2,40087,1732486423713 2024-11-24T22:15:45,751 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40087 {}] ipc.CallRunner(138): callId: 102 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:32832 deadline: 1732486605751, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94f3ccf9f8a638fa974e1f69627832ef, server=bd6592c37bb2,40087,1732486423713 2024-11-24T22:15:45,751 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40087 {}] ipc.CallRunner(138): callId: 118 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:32846 deadline: 1732486605751, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94f3ccf9f8a638fa974e1f69627832ef, server=bd6592c37bb2,40087,1732486423713 2024-11-24T22:15:45,751 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732486521610.94f3ccf9f8a638fa974e1f69627832ef., hostname=bd6592c37bb2,40087,1732486423713, seqNum=5 , the old value is region=TestAcidGuarantees,,1732486521610.94f3ccf9f8a638fa974e1f69627832ef., hostname=bd6592c37bb2,40087,1732486423713, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94f3ccf9f8a638fa974e1f69627832ef, server=bd6592c37bb2,40087,1732486423713 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T22:15:45,751 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732486521610.94f3ccf9f8a638fa974e1f69627832ef., hostname=bd6592c37bb2,40087,1732486423713, seqNum=5 , the old value is region=TestAcidGuarantees,,1732486521610.94f3ccf9f8a638fa974e1f69627832ef., hostname=bd6592c37bb2,40087,1732486423713, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94f3ccf9f8a638fa974e1f69627832ef, server=bd6592c37bb2,40087,1732486423713 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T22:15:45,751 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732486521610.94f3ccf9f8a638fa974e1f69627832ef., hostname=bd6592c37bb2,40087,1732486423713, seqNum=5 , the old value is region=TestAcidGuarantees,,1732486521610.94f3ccf9f8a638fa974e1f69627832ef., hostname=bd6592c37bb2,40087,1732486423713, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94f3ccf9f8a638fa974e1f69627832ef, server=bd6592c37bb2,40087,1732486423713 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T22:15:45,751 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732486521610.94f3ccf9f8a638fa974e1f69627832ef., hostname=bd6592c37bb2,40087,1732486423713, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94f3ccf9f8a638fa974e1f69627832ef, server=bd6592c37bb2,40087,1732486423713 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T22:15:45,752 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732486521610.94f3ccf9f8a638fa974e1f69627832ef., hostname=bd6592c37bb2,40087,1732486423713, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94f3ccf9f8a638fa974e1f69627832ef, server=bd6592c37bb2,40087,1732486423713 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T22:15:45,752 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732486521610.94f3ccf9f8a638fa974e1f69627832ef., hostname=bd6592c37bb2,40087,1732486423713, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94f3ccf9f8a638fa974e1f69627832ef, server=bd6592c37bb2,40087,1732486423713 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T22:15:45,752 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732486521610.94f3ccf9f8a638fa974e1f69627832ef., hostname=bd6592c37bb2,40087,1732486423713, seqNum=5 because the exception is null or not the one we care about 2024-11-24T22:15:45,752 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732486521610.94f3ccf9f8a638fa974e1f69627832ef., hostname=bd6592c37bb2,40087,1732486423713, seqNum=5 because the exception is null or not the one we care about 2024-11-24T22:15:45,752 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732486521610.94f3ccf9f8a638fa974e1f69627832ef., hostname=bd6592c37bb2,40087,1732486423713, seqNum=5 because the exception is null or not the one we care about 2024-11-24T22:15:46,063 INFO [RS_FLUSH_OPERATIONS-regionserver/bd6592c37bb2:0-2 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=291 (bloomFilter=true), to=hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/94f3ccf9f8a638fa974e1f69627832ef/.tmp/C/d8e07a696b374dd38a0665746ee004b3 2024-11-24T22:15:46,066 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bd6592c37bb2:0-2 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/94f3ccf9f8a638fa974e1f69627832ef/.tmp/A/510a774ece96416f8abbaee4f4c581cd as hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/94f3ccf9f8a638fa974e1f69627832ef/A/510a774ece96416f8abbaee4f4c581cd 2024-11-24T22:15:46,069 INFO [RS_FLUSH_OPERATIONS-regionserver/bd6592c37bb2:0-2 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/94f3ccf9f8a638fa974e1f69627832ef/A/510a774ece96416f8abbaee4f4c581cd, entries=150, sequenceid=291, filesize=30.5 K 2024-11-24T22:15:46,070 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bd6592c37bb2:0-2 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/94f3ccf9f8a638fa974e1f69627832ef/.tmp/B/32aece1bb9144ffb8c84c5135450995b as hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/94f3ccf9f8a638fa974e1f69627832ef/B/32aece1bb9144ffb8c84c5135450995b 2024-11-24T22:15:46,073 INFO [RS_FLUSH_OPERATIONS-regionserver/bd6592c37bb2:0-2 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/94f3ccf9f8a638fa974e1f69627832ef/B/32aece1bb9144ffb8c84c5135450995b, entries=150, sequenceid=291, filesize=12.0 K 2024-11-24T22:15:46,073 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bd6592c37bb2:0-2 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/94f3ccf9f8a638fa974e1f69627832ef/.tmp/C/d8e07a696b374dd38a0665746ee004b3 as hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/94f3ccf9f8a638fa974e1f69627832ef/C/d8e07a696b374dd38a0665746ee004b3 2024-11-24T22:15:46,075 INFO [RS_FLUSH_OPERATIONS-regionserver/bd6592c37bb2:0-2 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/94f3ccf9f8a638fa974e1f69627832ef/C/d8e07a696b374dd38a0665746ee004b3, entries=150, sequenceid=291, filesize=12.0 K 2024-11-24T22:15:46,076 INFO [RS_FLUSH_OPERATIONS-regionserver/bd6592c37bb2:0-2 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.HRegion(3140): Finished flush of dataSize ~67.09 KB/68700, heapSize ~199.92 KB/204720, currentSize=107.34 KB/109920 for 94f3ccf9f8a638fa974e1f69627832ef in 1649ms, sequenceid=291, compaction requested=true 2024-11-24T22:15:46,076 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bd6592c37bb2:0-2 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.HRegion(2603): Flush status journal for 94f3ccf9f8a638fa974e1f69627832ef: 2024-11-24T22:15:46,076 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bd6592c37bb2:0-2 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732486521610.94f3ccf9f8a638fa974e1f69627832ef. 2024-11-24T22:15:46,076 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bd6592c37bb2:0-2 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=112 2024-11-24T22:15:46,077 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37701 {}] master.HMaster(4169): Remote procedure done, pid=112 2024-11-24T22:15:46,079 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=112, resume processing ppid=111 2024-11-24T22:15:46,079 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=112, ppid=111, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.8030 sec 2024-11-24T22:15:46,080 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=111, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=111, table=TestAcidGuarantees in 1.8070 sec 2024-11-24T22:15:46,420 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37701 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=111 2024-11-24T22:15:46,421 INFO [RPCClient-NioEventLoopGroup-4-8 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-24T22:15:46,763 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40087 {}] regionserver.HRegion(8855): Flush requested on 94f3ccf9f8a638fa974e1f69627832ef 2024-11-24T22:15:46,763 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 94f3ccf9f8a638fa974e1f69627832ef 3/3 column families, dataSize=127.47 KB heapSize=379.97 KB 2024-11-24T22:15:46,763 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 94f3ccf9f8a638fa974e1f69627832ef, store=A 2024-11-24T22:15:46,763 INFO [Thread-1603 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1603. 2024-11-24T22:15:46,763 INFO [Thread-1595 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1595. 2024-11-24T22:15:46,763 DEBUG [Thread-1603 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-24T22:15:46,763 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-24T22:15:46,763 DEBUG [Thread-1595 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-24T22:15:46,763 DEBUG [Thread-1603 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-24T22:15:46,763 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 94f3ccf9f8a638fa974e1f69627832ef, store=B 2024-11-24T22:15:46,763 DEBUG [Thread-1595 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-24T22:15:46,763 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-24T22:15:46,763 INFO [Thread-1597 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1597. 2024-11-24T22:15:46,763 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 94f3ccf9f8a638fa974e1f69627832ef, store=C 2024-11-24T22:15:46,763 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-24T22:15:46,763 DEBUG [Thread-1597 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-24T22:15:46,763 DEBUG [Thread-1603 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-24T22:15:46,763 DEBUG [Thread-1597 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-24T22:15:46,763 DEBUG [Thread-1595 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-24T22:15:46,763 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-24T22:15:46,763 DEBUG [Thread-1597 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-24T22:15:46,764 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-24T22:15:46,764 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-24T22:15:46,769 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024112483afbdca7932409c9c912608cdcdfbb3_94f3ccf9f8a638fa974e1f69627832ef is 50, key is test_row_0/A:col10/1732486546761/Put/seqid=0 2024-11-24T22:15:46,772 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45929 is added to blk_1073742264_1440 (size=12454) 2024-11-24T22:15:47,173 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T22:15:47,176 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024112483afbdca7932409c9c912608cdcdfbb3_94f3ccf9f8a638fa974e1f69627832ef to hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024112483afbdca7932409c9c912608cdcdfbb3_94f3ccf9f8a638fa974e1f69627832ef 2024-11-24T22:15:47,176 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/94f3ccf9f8a638fa974e1f69627832ef/.tmp/A/0aaf705b0aec416c8bee1e69de45a7d4, store: [table=TestAcidGuarantees family=A region=94f3ccf9f8a638fa974e1f69627832ef] 2024-11-24T22:15:47,177 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/94f3ccf9f8a638fa974e1f69627832ef/.tmp/A/0aaf705b0aec416c8bee1e69de45a7d4 is 175, key is test_row_0/A:col10/1732486546761/Put/seqid=0 2024-11-24T22:15:47,180 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45929 is added to blk_1073742265_1441 (size=31255) 2024-11-24T22:15:47,580 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=313, memsize=42.5 K, hasBloomFilter=true, into tmp file hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/94f3ccf9f8a638fa974e1f69627832ef/.tmp/A/0aaf705b0aec416c8bee1e69de45a7d4 2024-11-24T22:15:47,586 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/94f3ccf9f8a638fa974e1f69627832ef/.tmp/B/a46e3bf0d2864cdf9c09d1df707b7fa9 is 50, key is test_row_0/B:col10/1732486546761/Put/seqid=0 2024-11-24T22:15:47,589 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45929 is added to blk_1073742266_1442 (size=12301) 2024-11-24T22:15:47,989 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=42.49 KB at sequenceid=313 (bloomFilter=true), to=hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/94f3ccf9f8a638fa974e1f69627832ef/.tmp/B/a46e3bf0d2864cdf9c09d1df707b7fa9 2024-11-24T22:15:47,997 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/94f3ccf9f8a638fa974e1f69627832ef/.tmp/C/d953442a120044119d80665c2d2c97c0 is 50, key is test_row_0/C:col10/1732486546761/Put/seqid=0 2024-11-24T22:15:47,999 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45929 is added to blk_1073742267_1443 (size=12301) 2024-11-24T22:15:48,400 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=42.49 KB at sequenceid=313 (bloomFilter=true), to=hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/94f3ccf9f8a638fa974e1f69627832ef/.tmp/C/d953442a120044119d80665c2d2c97c0 2024-11-24T22:15:48,404 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/94f3ccf9f8a638fa974e1f69627832ef/.tmp/A/0aaf705b0aec416c8bee1e69de45a7d4 as hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/94f3ccf9f8a638fa974e1f69627832ef/A/0aaf705b0aec416c8bee1e69de45a7d4 2024-11-24T22:15:48,407 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/94f3ccf9f8a638fa974e1f69627832ef/A/0aaf705b0aec416c8bee1e69de45a7d4, entries=150, sequenceid=313, filesize=30.5 K 2024-11-24T22:15:48,407 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/94f3ccf9f8a638fa974e1f69627832ef/.tmp/B/a46e3bf0d2864cdf9c09d1df707b7fa9 as hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/94f3ccf9f8a638fa974e1f69627832ef/B/a46e3bf0d2864cdf9c09d1df707b7fa9 2024-11-24T22:15:48,409 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/94f3ccf9f8a638fa974e1f69627832ef/B/a46e3bf0d2864cdf9c09d1df707b7fa9, entries=150, sequenceid=313, filesize=12.0 K 2024-11-24T22:15:48,410 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/94f3ccf9f8a638fa974e1f69627832ef/.tmp/C/d953442a120044119d80665c2d2c97c0 as hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/94f3ccf9f8a638fa974e1f69627832ef/C/d953442a120044119d80665c2d2c97c0 2024-11-24T22:15:48,413 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/94f3ccf9f8a638fa974e1f69627832ef/C/d953442a120044119d80665c2d2c97c0, entries=150, sequenceid=313, filesize=12.0 K 2024-11-24T22:15:48,414 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~127.47 KB/130530, heapSize ~379.92 KB/389040, currentSize=0 B/0 for 94f3ccf9f8a638fa974e1f69627832ef in 1651ms, sequenceid=313, compaction requested=true 2024-11-24T22:15:48,414 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 94f3ccf9f8a638fa974e1f69627832ef: 2024-11-24T22:15:48,414 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 94f3ccf9f8a638fa974e1f69627832ef:A, priority=-2147483648, current under compaction store size is 1 2024-11-24T22:15:48,414 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-24T22:15:48,414 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 94f3ccf9f8a638fa974e1f69627832ef:B, priority=-2147483648, current under compaction store size is 2 2024-11-24T22:15:48,414 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-24T22:15:48,414 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 94f3ccf9f8a638fa974e1f69627832ef:C, priority=-2147483648, current under compaction store size is 3 2024-11-24T22:15:48,414 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-24T22:15:48,414 DEBUG [RS:0;bd6592c37bb2:40087-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 7 store files, 0 compacting, 7 eligible, 16 blocking 2024-11-24T22:15:48,414 DEBUG [RS:0;bd6592c37bb2:40087-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 7 store files, 0 compacting, 7 eligible, 16 blocking 2024-11-24T22:15:48,415 DEBUG [RS:0;bd6592c37bb2:40087-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 7 files of size 86103 starting at candidate #0 after considering 15 permutations with 15 in ratio 2024-11-24T22:15:48,415 DEBUG [RS:0;bd6592c37bb2:40087-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 7 files of size 218781 starting at candidate #0 after considering 15 permutations with 15 in ratio 2024-11-24T22:15:48,415 DEBUG [RS:0;bd6592c37bb2:40087-longCompactions-0 {}] regionserver.HStore(1541): 94f3ccf9f8a638fa974e1f69627832ef/B is initiating minor compaction (all files) 2024-11-24T22:15:48,415 DEBUG [RS:0;bd6592c37bb2:40087-shortCompactions-0 {}] regionserver.HStore(1541): 94f3ccf9f8a638fa974e1f69627832ef/A is initiating minor compaction (all files) 2024-11-24T22:15:48,415 INFO [RS:0;bd6592c37bb2:40087-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 94f3ccf9f8a638fa974e1f69627832ef/A in TestAcidGuarantees,,1732486521610.94f3ccf9f8a638fa974e1f69627832ef. 2024-11-24T22:15:48,415 INFO [RS:0;bd6592c37bb2:40087-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 94f3ccf9f8a638fa974e1f69627832ef/B in TestAcidGuarantees,,1732486521610.94f3ccf9f8a638fa974e1f69627832ef. 2024-11-24T22:15:48,415 INFO [RS:0;bd6592c37bb2:40087-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/94f3ccf9f8a638fa974e1f69627832ef/A/090547ea69e9495fb9b887db92c6c3da, hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/94f3ccf9f8a638fa974e1f69627832ef/A/93e608ac185c42b39eac1782462a5302, hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/94f3ccf9f8a638fa974e1f69627832ef/A/57308f1fd93a41fa954bec339b0cb607, hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/94f3ccf9f8a638fa974e1f69627832ef/A/8d09d43826484065b9293366a1476cb5, hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/94f3ccf9f8a638fa974e1f69627832ef/A/19c3184bc0c8467ea8a35a00f8b1bd84, hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/94f3ccf9f8a638fa974e1f69627832ef/A/510a774ece96416f8abbaee4f4c581cd, hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/94f3ccf9f8a638fa974e1f69627832ef/A/0aaf705b0aec416c8bee1e69de45a7d4] into tmpdir=hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/94f3ccf9f8a638fa974e1f69627832ef/.tmp, totalSize=213.7 K 2024-11-24T22:15:48,415 INFO [RS:0;bd6592c37bb2:40087-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/94f3ccf9f8a638fa974e1f69627832ef/B/3ef534ba6992421fb44a53ef9a12e571, hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/94f3ccf9f8a638fa974e1f69627832ef/B/5cbd879d83374485b483f18f6194ad70, hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/94f3ccf9f8a638fa974e1f69627832ef/B/149583da245f45179d6de44960e3e48e, hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/94f3ccf9f8a638fa974e1f69627832ef/B/154e2a22f31844d198f56aa1a33d20ca, hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/94f3ccf9f8a638fa974e1f69627832ef/B/472d5538cdba45b3ae87ece46ad44a5e, hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/94f3ccf9f8a638fa974e1f69627832ef/B/32aece1bb9144ffb8c84c5135450995b, hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/94f3ccf9f8a638fa974e1f69627832ef/B/a46e3bf0d2864cdf9c09d1df707b7fa9] into tmpdir=hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/94f3ccf9f8a638fa974e1f69627832ef/.tmp, totalSize=84.1 K 2024-11-24T22:15:48,415 INFO [RS:0;bd6592c37bb2:40087-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=9 throughput controller=DefaultCompactionThroughputController [maxThroughput=57.69 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1732486521610.94f3ccf9f8a638fa974e1f69627832ef. 2024-11-24T22:15:48,416 DEBUG [RS:0;bd6592c37bb2:40087-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1732486521610.94f3ccf9f8a638fa974e1f69627832ef. files: [hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/94f3ccf9f8a638fa974e1f69627832ef/A/090547ea69e9495fb9b887db92c6c3da, hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/94f3ccf9f8a638fa974e1f69627832ef/A/93e608ac185c42b39eac1782462a5302, hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/94f3ccf9f8a638fa974e1f69627832ef/A/57308f1fd93a41fa954bec339b0cb607, hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/94f3ccf9f8a638fa974e1f69627832ef/A/8d09d43826484065b9293366a1476cb5, hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/94f3ccf9f8a638fa974e1f69627832ef/A/19c3184bc0c8467ea8a35a00f8b1bd84, hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/94f3ccf9f8a638fa974e1f69627832ef/A/510a774ece96416f8abbaee4f4c581cd, hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/94f3ccf9f8a638fa974e1f69627832ef/A/0aaf705b0aec416c8bee1e69de45a7d4] 2024-11-24T22:15:48,416 DEBUG [RS:0;bd6592c37bb2:40087-longCompactions-0 {}] compactions.Compactor(225): Compacting 3ef534ba6992421fb44a53ef9a12e571, keycount=150, bloomtype=ROW, size=12.4 K, encoding=NONE, compression=NONE, seqNum=211, earliestPutTs=1732486537048 2024-11-24T22:15:48,416 DEBUG [RS:0;bd6592c37bb2:40087-shortCompactions-0 {}] compactions.Compactor(225): Compacting 090547ea69e9495fb9b887db92c6c3da, keycount=150, bloomtype=ROW, size=30.9 K, encoding=NONE, compression=NONE, seqNum=211, earliestPutTs=1732486537048 2024-11-24T22:15:48,416 DEBUG [RS:0;bd6592c37bb2:40087-longCompactions-0 {}] compactions.Compactor(225): Compacting 5cbd879d83374485b483f18f6194ad70, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=224, earliestPutTs=1732486538257 2024-11-24T22:15:48,416 DEBUG [RS:0;bd6592c37bb2:40087-shortCompactions-0 {}] compactions.Compactor(225): Compacting 93e608ac185c42b39eac1782462a5302, keycount=150, bloomtype=ROW, size=30.4 K, encoding=NONE, compression=NONE, seqNum=224, earliestPutTs=1732486538257 2024-11-24T22:15:48,416 DEBUG [RS:0;bd6592c37bb2:40087-longCompactions-0 {}] compactions.Compactor(225): Compacting 149583da245f45179d6de44960e3e48e, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=246, earliestPutTs=1732486539479 2024-11-24T22:15:48,416 DEBUG [RS:0;bd6592c37bb2:40087-shortCompactions-0 {}] compactions.Compactor(225): Compacting 57308f1fd93a41fa954bec339b0cb607, keycount=150, bloomtype=ROW, size=30.4 K, encoding=NONE, compression=NONE, seqNum=246, earliestPutTs=1732486539479 2024-11-24T22:15:48,417 DEBUG [RS:0;bd6592c37bb2:40087-longCompactions-0 {}] compactions.Compactor(225): Compacting 154e2a22f31844d198f56aa1a33d20ca, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=259, earliestPutTs=1732486541712 2024-11-24T22:15:48,417 DEBUG [RS:0;bd6592c37bb2:40087-shortCompactions-0 {}] compactions.Compactor(225): Compacting 8d09d43826484065b9293366a1476cb5, keycount=150, bloomtype=ROW, size=30.4 K, encoding=NONE, compression=NONE, seqNum=259, earliestPutTs=1732486541712 2024-11-24T22:15:48,417 DEBUG [RS:0;bd6592c37bb2:40087-longCompactions-0 {}] compactions.Compactor(225): Compacting 472d5538cdba45b3ae87ece46ad44a5e, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=278, earliestPutTs=1732486542921 2024-11-24T22:15:48,417 DEBUG [RS:0;bd6592c37bb2:40087-shortCompactions-0 {}] compactions.Compactor(225): Compacting 19c3184bc0c8467ea8a35a00f8b1bd84, keycount=150, bloomtype=ROW, size=30.5 K, encoding=NONE, compression=NONE, seqNum=278, earliestPutTs=1732486542921 2024-11-24T22:15:48,417 DEBUG [RS:0;bd6592c37bb2:40087-longCompactions-0 {}] compactions.Compactor(225): Compacting 32aece1bb9144ffb8c84c5135450995b, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=291, earliestPutTs=1732486543306 2024-11-24T22:15:48,417 DEBUG [RS:0;bd6592c37bb2:40087-shortCompactions-0 {}] compactions.Compactor(225): Compacting 510a774ece96416f8abbaee4f4c581cd, keycount=150, bloomtype=ROW, size=30.5 K, encoding=NONE, compression=NONE, seqNum=291, earliestPutTs=1732486543306 2024-11-24T22:15:48,417 DEBUG [RS:0;bd6592c37bb2:40087-longCompactions-0 {}] compactions.Compactor(225): Compacting a46e3bf0d2864cdf9c09d1df707b7fa9, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=313, earliestPutTs=1732486544590 2024-11-24T22:15:48,417 DEBUG [RS:0;bd6592c37bb2:40087-shortCompactions-0 {}] compactions.Compactor(225): Compacting 0aaf705b0aec416c8bee1e69de45a7d4, keycount=150, bloomtype=ROW, size=30.5 K, encoding=NONE, compression=NONE, seqNum=313, earliestPutTs=1732486544590 2024-11-24T22:15:48,426 INFO [RS:0;bd6592c37bb2:40087-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=94f3ccf9f8a638fa974e1f69627832ef] 2024-11-24T22:15:48,427 DEBUG [RS:0;bd6592c37bb2:40087-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e202411244546976becc6404b8a0eeb1b01940593_94f3ccf9f8a638fa974e1f69627832ef store=[table=TestAcidGuarantees family=A region=94f3ccf9f8a638fa974e1f69627832ef] 2024-11-24T22:15:48,428 INFO [RS:0;bd6592c37bb2:40087-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 94f3ccf9f8a638fa974e1f69627832ef#B#compaction#368 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 57.69 MB/second 2024-11-24T22:15:48,428 DEBUG [RS:0;bd6592c37bb2:40087-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/94f3ccf9f8a638fa974e1f69627832ef/.tmp/B/836bd1834e214114983e4a2790195396 is 50, key is test_row_0/B:col10/1732486546761/Put/seqid=0 2024-11-24T22:15:48,431 DEBUG [RS:0;bd6592c37bb2:40087-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e202411244546976becc6404b8a0eeb1b01940593_94f3ccf9f8a638fa974e1f69627832ef, store=[table=TestAcidGuarantees family=A region=94f3ccf9f8a638fa974e1f69627832ef] 2024-11-24T22:15:48,431 DEBUG [RS:0;bd6592c37bb2:40087-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411244546976becc6404b8a0eeb1b01940593_94f3ccf9f8a638fa974e1f69627832ef because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=94f3ccf9f8a638fa974e1f69627832ef] 2024-11-24T22:15:48,432 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45929 is added to blk_1073742268_1444 (size=12541) 2024-11-24T22:15:48,437 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45929 is added to blk_1073742269_1445 (size=4469) 2024-11-24T22:15:48,836 DEBUG [RS:0;bd6592c37bb2:40087-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/94f3ccf9f8a638fa974e1f69627832ef/.tmp/B/836bd1834e214114983e4a2790195396 as hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/94f3ccf9f8a638fa974e1f69627832ef/B/836bd1834e214114983e4a2790195396 2024-11-24T22:15:48,838 INFO [RS:0;bd6592c37bb2:40087-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 94f3ccf9f8a638fa974e1f69627832ef#A#compaction#369 average throughput is 0.06 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 57.69 MB/second 2024-11-24T22:15:48,838 DEBUG [RS:0;bd6592c37bb2:40087-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/94f3ccf9f8a638fa974e1f69627832ef/.tmp/A/d968deefbf564bcc9d4adf3de5e8cd45 is 175, key is test_row_0/A:col10/1732486546761/Put/seqid=0 2024-11-24T22:15:48,840 INFO [RS:0;bd6592c37bb2:40087-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 7 (all) file(s) in 94f3ccf9f8a638fa974e1f69627832ef/B of 94f3ccf9f8a638fa974e1f69627832ef into 836bd1834e214114983e4a2790195396(size=12.2 K), total size for store is 12.2 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-24T22:15:48,840 DEBUG [RS:0;bd6592c37bb2:40087-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 94f3ccf9f8a638fa974e1f69627832ef: 2024-11-24T22:15:48,840 INFO [RS:0;bd6592c37bb2:40087-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732486521610.94f3ccf9f8a638fa974e1f69627832ef., storeName=94f3ccf9f8a638fa974e1f69627832ef/B, priority=9, startTime=1732486548414; duration=0sec 2024-11-24T22:15:48,840 DEBUG [RS:0;bd6592c37bb2:40087-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-24T22:15:48,840 DEBUG [RS:0;bd6592c37bb2:40087-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 94f3ccf9f8a638fa974e1f69627832ef:B 2024-11-24T22:15:48,841 DEBUG [RS:0;bd6592c37bb2:40087-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 7 store files, 0 compacting, 7 eligible, 16 blocking 2024-11-24T22:15:48,842 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45929 is added to blk_1073742270_1446 (size=31495) 2024-11-24T22:15:48,842 DEBUG [RS:0;bd6592c37bb2:40087-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 7 files of size 86069 starting at candidate #0 after considering 15 permutations with 15 in ratio 2024-11-24T22:15:48,842 DEBUG [RS:0;bd6592c37bb2:40087-longCompactions-0 {}] regionserver.HStore(1541): 94f3ccf9f8a638fa974e1f69627832ef/C is initiating minor compaction (all files) 2024-11-24T22:15:48,842 INFO [RS:0;bd6592c37bb2:40087-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 94f3ccf9f8a638fa974e1f69627832ef/C in TestAcidGuarantees,,1732486521610.94f3ccf9f8a638fa974e1f69627832ef. 2024-11-24T22:15:48,842 INFO [RS:0;bd6592c37bb2:40087-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/94f3ccf9f8a638fa974e1f69627832ef/C/33635f7b1a774316b05468b61e43d862, hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/94f3ccf9f8a638fa974e1f69627832ef/C/25ca7dfcdf7c4069ba937e2d7f4fc7e7, hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/94f3ccf9f8a638fa974e1f69627832ef/C/62d1d497a09e4e0fb4316beebe854732, hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/94f3ccf9f8a638fa974e1f69627832ef/C/73ea5ac0072e48f180abd4ef66e7d3fe, hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/94f3ccf9f8a638fa974e1f69627832ef/C/4920a0a11aed4fdfbbaa49f3afcc84b1, hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/94f3ccf9f8a638fa974e1f69627832ef/C/d8e07a696b374dd38a0665746ee004b3, hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/94f3ccf9f8a638fa974e1f69627832ef/C/d953442a120044119d80665c2d2c97c0] into tmpdir=hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/94f3ccf9f8a638fa974e1f69627832ef/.tmp, totalSize=84.1 K 2024-11-24T22:15:48,843 DEBUG [RS:0;bd6592c37bb2:40087-longCompactions-0 {}] compactions.Compactor(225): Compacting 33635f7b1a774316b05468b61e43d862, keycount=150, bloomtype=ROW, size=12.4 K, encoding=NONE, compression=NONE, seqNum=211, earliestPutTs=1732486537048 2024-11-24T22:15:48,843 DEBUG [RS:0;bd6592c37bb2:40087-longCompactions-0 {}] compactions.Compactor(225): Compacting 25ca7dfcdf7c4069ba937e2d7f4fc7e7, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=224, earliestPutTs=1732486538257 2024-11-24T22:15:48,843 DEBUG [RS:0;bd6592c37bb2:40087-longCompactions-0 {}] compactions.Compactor(225): Compacting 62d1d497a09e4e0fb4316beebe854732, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=246, earliestPutTs=1732486539479 2024-11-24T22:15:48,843 DEBUG [RS:0;bd6592c37bb2:40087-longCompactions-0 {}] compactions.Compactor(225): Compacting 73ea5ac0072e48f180abd4ef66e7d3fe, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=259, earliestPutTs=1732486541712 2024-11-24T22:15:48,844 DEBUG [RS:0;bd6592c37bb2:40087-longCompactions-0 {}] compactions.Compactor(225): Compacting 4920a0a11aed4fdfbbaa49f3afcc84b1, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=278, earliestPutTs=1732486542921 2024-11-24T22:15:48,844 DEBUG [RS:0;bd6592c37bb2:40087-longCompactions-0 {}] compactions.Compactor(225): Compacting d8e07a696b374dd38a0665746ee004b3, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=291, earliestPutTs=1732486543306 2024-11-24T22:15:48,844 DEBUG [RS:0;bd6592c37bb2:40087-longCompactions-0 {}] compactions.Compactor(225): Compacting d953442a120044119d80665c2d2c97c0, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=313, earliestPutTs=1732486544590 2024-11-24T22:15:48,854 INFO [RS:0;bd6592c37bb2:40087-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 94f3ccf9f8a638fa974e1f69627832ef#C#compaction#370 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 57.69 MB/second 2024-11-24T22:15:48,854 DEBUG [RS:0;bd6592c37bb2:40087-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/94f3ccf9f8a638fa974e1f69627832ef/.tmp/C/dabbccd5526b4836aa77269cf112f130 is 50, key is test_row_0/C:col10/1732486546761/Put/seqid=0 2024-11-24T22:15:48,857 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45929 is added to blk_1073742271_1447 (size=12541) 2024-11-24T22:15:49,246 DEBUG [RS:0;bd6592c37bb2:40087-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/94f3ccf9f8a638fa974e1f69627832ef/.tmp/A/d968deefbf564bcc9d4adf3de5e8cd45 as hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/94f3ccf9f8a638fa974e1f69627832ef/A/d968deefbf564bcc9d4adf3de5e8cd45 2024-11-24T22:15:49,249 INFO [RS:0;bd6592c37bb2:40087-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 7 (all) file(s) in 94f3ccf9f8a638fa974e1f69627832ef/A of 94f3ccf9f8a638fa974e1f69627832ef into d968deefbf564bcc9d4adf3de5e8cd45(size=30.8 K), total size for store is 30.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-24T22:15:49,249 DEBUG [RS:0;bd6592c37bb2:40087-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 94f3ccf9f8a638fa974e1f69627832ef: 2024-11-24T22:15:49,249 INFO [RS:0;bd6592c37bb2:40087-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732486521610.94f3ccf9f8a638fa974e1f69627832ef., storeName=94f3ccf9f8a638fa974e1f69627832ef/A, priority=9, startTime=1732486548414; duration=0sec 2024-11-24T22:15:49,249 DEBUG [RS:0;bd6592c37bb2:40087-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-24T22:15:49,249 DEBUG [RS:0;bd6592c37bb2:40087-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 94f3ccf9f8a638fa974e1f69627832ef:A 2024-11-24T22:15:49,260 DEBUG [RS:0;bd6592c37bb2:40087-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/94f3ccf9f8a638fa974e1f69627832ef/.tmp/C/dabbccd5526b4836aa77269cf112f130 as hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/94f3ccf9f8a638fa974e1f69627832ef/C/dabbccd5526b4836aa77269cf112f130 2024-11-24T22:15:49,263 INFO [RS:0;bd6592c37bb2:40087-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 7 (all) file(s) in 94f3ccf9f8a638fa974e1f69627832ef/C of 94f3ccf9f8a638fa974e1f69627832ef into dabbccd5526b4836aa77269cf112f130(size=12.2 K), total size for store is 12.2 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-24T22:15:49,263 DEBUG [RS:0;bd6592c37bb2:40087-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 94f3ccf9f8a638fa974e1f69627832ef: 2024-11-24T22:15:49,264 INFO [RS:0;bd6592c37bb2:40087-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732486521610.94f3ccf9f8a638fa974e1f69627832ef., storeName=94f3ccf9f8a638fa974e1f69627832ef/C, priority=9, startTime=1732486548414; duration=0sec 2024-11-24T22:15:49,264 DEBUG [RS:0;bd6592c37bb2:40087-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-24T22:15:49,264 DEBUG [RS:0;bd6592c37bb2:40087-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 94f3ccf9f8a638fa974e1f69627832ef:C 2024-11-24T22:15:53,032 INFO [Thread-1599 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1599. 2024-11-24T22:15:53,032 DEBUG [Thread-1599 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-24T22:15:53,032 DEBUG [Thread-1599 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-24T22:15:53,032 DEBUG [Thread-1599 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-24T22:15:53,032 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-24T22:15:54,052 INFO [Thread-1601 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1601. 2024-11-24T22:15:54,052 DEBUG [Thread-1601 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-24T22:15:54,052 DEBUG [Thread-1601 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-24T22:15:54,052 DEBUG [Thread-1601 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-24T22:15:54,052 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-24T22:15:54,053 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(392): Finished test. Writers: 2024-11-24T22:15:54,053 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 50 2024-11-24T22:15:54,053 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 60 2024-11-24T22:15:54,053 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 28 2024-11-24T22:15:54,053 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 46 2024-11-24T22:15:54,053 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 59 2024-11-24T22:15:54,053 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(396): Readers: 2024-11-24T22:15:54,053 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(400): Scanners: 2024-11-24T22:15:54,053 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(402): scanned 1048 2024-11-24T22:15:54,053 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(403): verified 3140 rows 2024-11-24T22:15:54,053 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(402): scanned 1052 2024-11-24T22:15:54,053 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(403): verified 3149 rows 2024-11-24T22:15:54,053 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(402): scanned 1048 2024-11-24T22:15:54,053 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(403): verified 3143 rows 2024-11-24T22:15:54,053 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(402): scanned 1050 2024-11-24T22:15:54,053 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(403): verified 3142 rows 2024-11-24T22:15:54,053 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(402): scanned 1049 2024-11-24T22:15:54,053 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(403): verified 3139 rows 2024-11-24T22:15:54,053 INFO [Time-limited test {}] client.AsyncConnectionImpl(233): Connection has been closed by Time-limited test. 2024-11-24T22:15:54,053 DEBUG [Time-limited test {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool.doWork(AcidGuaranteesTestTool.java:131) at org.apache.hadoop.hbase.util.AbstractHBaseTool.run(AbstractHBaseTool.java:150) at org.apache.hadoop.hbase.AcidGuaranteesTestBase.runTestAtomicity(AcidGuaranteesTestBase.java:102) at org.apache.hadoop.hbase.AcidGuaranteesTestBase.testMobScanAtomicity(AcidGuaranteesTestBase.java:127) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) at java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) at java.base/java.lang.reflect.Method.invoke(Method.java:568) at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) at java.base/java.util.concurrent.FutureTask.run(FutureTask.java:264) at java.base/java.lang.Thread.run(Thread.java:840) 2024-11-24T22:15:54,053 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-24T22:15:54,053 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-24T22:15:54,053 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-24T22:15:54,054 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37701 {}] master.HMaster$13(2820): Client=jenkins//172.17.0.2 disable TestAcidGuarantees 2024-11-24T22:15:54,055 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37701 {}] procedure2.ProcedureExecutor(1139): Stored pid=113, state=RUNNABLE:DISABLE_TABLE_PREPARE, hasLock=false; DisableTableProcedure table=TestAcidGuarantees 2024-11-24T22:15:54,056 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37701 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=113 2024-11-24T22:15:54,057 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1732486554057"}]},"ts":"1732486554057"} 2024-11-24T22:15:54,058 INFO [PEWorker-1 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=DISABLING in hbase:meta 2024-11-24T22:15:54,058 INFO [PEWorker-1 {}] procedure.DisableTableProcedure(284): Set TestAcidGuarantees to state=DISABLING 2024-11-24T22:15:54,059 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=114, ppid=113, state=RUNNABLE:CLOSE_TABLE_REGIONS_SCHEDULE, hasLock=false; CloseTableRegionsProcedure table=TestAcidGuarantees}] 2024-11-24T22:15:54,060 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=115, ppid=114, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=94f3ccf9f8a638fa974e1f69627832ef, UNASSIGN}] 2024-11-24T22:15:54,060 INFO [PEWorker-4 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=115, ppid=114, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=94f3ccf9f8a638fa974e1f69627832ef, UNASSIGN 2024-11-24T22:15:54,061 INFO [PEWorker-4 {}] assignment.RegionStateStore(223): pid=115 updating hbase:meta row=94f3ccf9f8a638fa974e1f69627832ef, regionState=CLOSING, regionLocation=bd6592c37bb2,40087,1732486423713 2024-11-24T22:15:54,062 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=115, ppid=114, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=94f3ccf9f8a638fa974e1f69627832ef, UNASSIGN because future has completed 2024-11-24T22:15:54,062 DEBUG [PEWorker-5 {}] assignment.TransitRegionStateProcedure(375): Close region: isSplit: false: evictOnSplit: true: evictOnClose: false 2024-11-24T22:15:54,062 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=116, ppid=115, state=RUNNABLE, hasLock=false; CloseRegionProcedure 94f3ccf9f8a638fa974e1f69627832ef, server=bd6592c37bb2,40087,1732486423713}] 2024-11-24T22:15:54,161 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37701 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=113 2024-11-24T22:15:54,214 INFO [RS_CLOSE_REGION-regionserver/bd6592c37bb2:0-0 {event_type=M_RS_CLOSE_REGION, pid=116}] handler.UnassignRegionHandler(122): Close 94f3ccf9f8a638fa974e1f69627832ef 2024-11-24T22:15:54,214 DEBUG [RS_CLOSE_REGION-regionserver/bd6592c37bb2:0-0 {event_type=M_RS_CLOSE_REGION, pid=116}] handler.UnassignRegionHandler(136): Unassign region: split region: false: evictCache: false 2024-11-24T22:15:54,214 DEBUG [RS_CLOSE_REGION-regionserver/bd6592c37bb2:0-0 {event_type=M_RS_CLOSE_REGION, pid=116}] regionserver.HRegion(1722): Closing 94f3ccf9f8a638fa974e1f69627832ef, disabling compactions & flushes 2024-11-24T22:15:54,214 INFO [RS_CLOSE_REGION-regionserver/bd6592c37bb2:0-0 {event_type=M_RS_CLOSE_REGION, pid=116}] regionserver.HRegion(1755): Closing region TestAcidGuarantees,,1732486521610.94f3ccf9f8a638fa974e1f69627832ef. 2024-11-24T22:15:54,214 DEBUG [RS_CLOSE_REGION-regionserver/bd6592c37bb2:0-0 {event_type=M_RS_CLOSE_REGION, pid=116}] regionserver.HRegion(1776): Time limited wait for close lock on TestAcidGuarantees,,1732486521610.94f3ccf9f8a638fa974e1f69627832ef. 2024-11-24T22:15:54,214 DEBUG [RS_CLOSE_REGION-regionserver/bd6592c37bb2:0-0 {event_type=M_RS_CLOSE_REGION, pid=116}] regionserver.HRegion(1843): Acquired close lock on TestAcidGuarantees,,1732486521610.94f3ccf9f8a638fa974e1f69627832ef. after waiting 0 ms 2024-11-24T22:15:54,214 DEBUG [RS_CLOSE_REGION-regionserver/bd6592c37bb2:0-0 {event_type=M_RS_CLOSE_REGION, pid=116}] regionserver.HRegion(1853): Updates disabled for region TestAcidGuarantees,,1732486521610.94f3ccf9f8a638fa974e1f69627832ef. 2024-11-24T22:15:54,214 INFO [RS_CLOSE_REGION-regionserver/bd6592c37bb2:0-0 {event_type=M_RS_CLOSE_REGION, pid=116}] regionserver.HRegion(2902): Flushing 94f3ccf9f8a638fa974e1f69627832ef 3/3 column families, dataSize=13.42 KB heapSize=40.59 KB 2024-11-24T22:15:54,215 DEBUG [RS_CLOSE_REGION-regionserver/bd6592c37bb2:0-0 {event_type=M_RS_CLOSE_REGION, pid=116}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 94f3ccf9f8a638fa974e1f69627832ef, store=A 2024-11-24T22:15:54,215 DEBUG [RS_CLOSE_REGION-regionserver/bd6592c37bb2:0-0 {event_type=M_RS_CLOSE_REGION, pid=116}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-24T22:15:54,215 DEBUG [RS_CLOSE_REGION-regionserver/bd6592c37bb2:0-0 {event_type=M_RS_CLOSE_REGION, pid=116}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 94f3ccf9f8a638fa974e1f69627832ef, store=B 2024-11-24T22:15:54,215 DEBUG [RS_CLOSE_REGION-regionserver/bd6592c37bb2:0-0 {event_type=M_RS_CLOSE_REGION, pid=116}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-24T22:15:54,215 DEBUG [RS_CLOSE_REGION-regionserver/bd6592c37bb2:0-0 {event_type=M_RS_CLOSE_REGION, pid=116}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 94f3ccf9f8a638fa974e1f69627832ef, store=C 2024-11-24T22:15:54,215 DEBUG [RS_CLOSE_REGION-regionserver/bd6592c37bb2:0-0 {event_type=M_RS_CLOSE_REGION, pid=116}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-24T22:15:54,223 DEBUG [RS_CLOSE_REGION-regionserver/bd6592c37bb2:0-0 {event_type=M_RS_CLOSE_REGION, pid=116}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411247fa1ae1fd2c54228a8ce550e28aa4a3d_94f3ccf9f8a638fa974e1f69627832ef is 50, key is test_row_2/A:col10/1732486554051/Put/seqid=0 2024-11-24T22:15:54,226 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45929 is added to blk_1073742272_1448 (size=7374) 2024-11-24T22:15:54,371 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37701 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=113 2024-11-24T22:15:54,627 DEBUG [RS_CLOSE_REGION-regionserver/bd6592c37bb2:0-0 {event_type=M_RS_CLOSE_REGION, pid=116}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T22:15:54,631 INFO [RS_CLOSE_REGION-regionserver/bd6592c37bb2:0-0 {event_type=M_RS_CLOSE_REGION, pid=116}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411247fa1ae1fd2c54228a8ce550e28aa4a3d_94f3ccf9f8a638fa974e1f69627832ef to hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411247fa1ae1fd2c54228a8ce550e28aa4a3d_94f3ccf9f8a638fa974e1f69627832ef 2024-11-24T22:15:54,632 DEBUG [RS_CLOSE_REGION-regionserver/bd6592c37bb2:0-0 {event_type=M_RS_CLOSE_REGION, pid=116}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/94f3ccf9f8a638fa974e1f69627832ef/.tmp/A/de25378ccb7a422d967b6c53762341c8, store: [table=TestAcidGuarantees family=A region=94f3ccf9f8a638fa974e1f69627832ef] 2024-11-24T22:15:54,632 DEBUG [RS_CLOSE_REGION-regionserver/bd6592c37bb2:0-0 {event_type=M_RS_CLOSE_REGION, pid=116}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/94f3ccf9f8a638fa974e1f69627832ef/.tmp/A/de25378ccb7a422d967b6c53762341c8 is 175, key is test_row_2/A:col10/1732486554051/Put/seqid=0 2024-11-24T22:15:54,635 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45929 is added to blk_1073742273_1449 (size=13865) 2024-11-24T22:15:54,681 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37701 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=113 2024-11-24T22:15:55,036 INFO [RS_CLOSE_REGION-regionserver/bd6592c37bb2:0-0 {event_type=M_RS_CLOSE_REGION, pid=116}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=321, memsize=4.5 K, hasBloomFilter=true, into tmp file hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/94f3ccf9f8a638fa974e1f69627832ef/.tmp/A/de25378ccb7a422d967b6c53762341c8 2024-11-24T22:15:55,042 DEBUG [RS_CLOSE_REGION-regionserver/bd6592c37bb2:0-0 {event_type=M_RS_CLOSE_REGION, pid=116}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/94f3ccf9f8a638fa974e1f69627832ef/.tmp/B/82a4e0bfd1514c5aa8c9fce35ee180f7 is 50, key is test_row_2/B:col10/1732486554051/Put/seqid=0 2024-11-24T22:15:55,044 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45929 is added to blk_1073742274_1450 (size=7415) 2024-11-24T22:15:55,190 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37701 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=113 2024-11-24T22:15:55,445 INFO [RS_CLOSE_REGION-regionserver/bd6592c37bb2:0-0 {event_type=M_RS_CLOSE_REGION, pid=116}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=4.47 KB at sequenceid=321 (bloomFilter=true), to=hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/94f3ccf9f8a638fa974e1f69627832ef/.tmp/B/82a4e0bfd1514c5aa8c9fce35ee180f7 2024-11-24T22:15:55,453 DEBUG [RS_CLOSE_REGION-regionserver/bd6592c37bb2:0-0 {event_type=M_RS_CLOSE_REGION, pid=116}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/94f3ccf9f8a638fa974e1f69627832ef/.tmp/C/37f2846949b4429bb215ef889bd8a7f0 is 50, key is test_row_2/C:col10/1732486554051/Put/seqid=0 2024-11-24T22:15:55,455 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45929 is added to blk_1073742275_1451 (size=7415) 2024-11-24T22:15:55,856 INFO [RS_CLOSE_REGION-regionserver/bd6592c37bb2:0-0 {event_type=M_RS_CLOSE_REGION, pid=116}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=4.47 KB at sequenceid=321 (bloomFilter=true), to=hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/94f3ccf9f8a638fa974e1f69627832ef/.tmp/C/37f2846949b4429bb215ef889bd8a7f0 2024-11-24T22:15:55,860 DEBUG [RS_CLOSE_REGION-regionserver/bd6592c37bb2:0-0 {event_type=M_RS_CLOSE_REGION, pid=116}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/94f3ccf9f8a638fa974e1f69627832ef/.tmp/A/de25378ccb7a422d967b6c53762341c8 as hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/94f3ccf9f8a638fa974e1f69627832ef/A/de25378ccb7a422d967b6c53762341c8 2024-11-24T22:15:55,863 INFO [RS_CLOSE_REGION-regionserver/bd6592c37bb2:0-0 {event_type=M_RS_CLOSE_REGION, pid=116}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/94f3ccf9f8a638fa974e1f69627832ef/A/de25378ccb7a422d967b6c53762341c8, entries=50, sequenceid=321, filesize=13.5 K 2024-11-24T22:15:55,863 DEBUG [RS_CLOSE_REGION-regionserver/bd6592c37bb2:0-0 {event_type=M_RS_CLOSE_REGION, pid=116}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/94f3ccf9f8a638fa974e1f69627832ef/.tmp/B/82a4e0bfd1514c5aa8c9fce35ee180f7 as hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/94f3ccf9f8a638fa974e1f69627832ef/B/82a4e0bfd1514c5aa8c9fce35ee180f7 2024-11-24T22:15:55,866 INFO [RS_CLOSE_REGION-regionserver/bd6592c37bb2:0-0 {event_type=M_RS_CLOSE_REGION, pid=116}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/94f3ccf9f8a638fa974e1f69627832ef/B/82a4e0bfd1514c5aa8c9fce35ee180f7, entries=50, sequenceid=321, filesize=7.2 K 2024-11-24T22:15:55,866 DEBUG [RS_CLOSE_REGION-regionserver/bd6592c37bb2:0-0 {event_type=M_RS_CLOSE_REGION, pid=116}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/94f3ccf9f8a638fa974e1f69627832ef/.tmp/C/37f2846949b4429bb215ef889bd8a7f0 as hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/94f3ccf9f8a638fa974e1f69627832ef/C/37f2846949b4429bb215ef889bd8a7f0 2024-11-24T22:15:55,869 INFO [RS_CLOSE_REGION-regionserver/bd6592c37bb2:0-0 {event_type=M_RS_CLOSE_REGION, pid=116}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/94f3ccf9f8a638fa974e1f69627832ef/C/37f2846949b4429bb215ef889bd8a7f0, entries=50, sequenceid=321, filesize=7.2 K 2024-11-24T22:15:55,870 INFO [RS_CLOSE_REGION-regionserver/bd6592c37bb2:0-0 {event_type=M_RS_CLOSE_REGION, pid=116}] regionserver.HRegion(3140): Finished flush of dataSize ~13.42 KB/13740, heapSize ~40.55 KB/41520, currentSize=0 B/0 for 94f3ccf9f8a638fa974e1f69627832ef in 1656ms, sequenceid=321, compaction requested=false 2024-11-24T22:15:55,870 DEBUG [StoreCloser-TestAcidGuarantees,,1732486521610.94f3ccf9f8a638fa974e1f69627832ef.-1 {}] regionserver.HStore(2317): Moving the files [hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/94f3ccf9f8a638fa974e1f69627832ef/A/090547ea69e9495fb9b887db92c6c3da, hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/94f3ccf9f8a638fa974e1f69627832ef/A/93e608ac185c42b39eac1782462a5302, hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/94f3ccf9f8a638fa974e1f69627832ef/A/57308f1fd93a41fa954bec339b0cb607, hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/94f3ccf9f8a638fa974e1f69627832ef/A/8d09d43826484065b9293366a1476cb5, hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/94f3ccf9f8a638fa974e1f69627832ef/A/19c3184bc0c8467ea8a35a00f8b1bd84, hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/94f3ccf9f8a638fa974e1f69627832ef/A/510a774ece96416f8abbaee4f4c581cd, hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/94f3ccf9f8a638fa974e1f69627832ef/A/0aaf705b0aec416c8bee1e69de45a7d4] to archive 2024-11-24T22:15:55,871 DEBUG [StoreCloser-TestAcidGuarantees,,1732486521610.94f3ccf9f8a638fa974e1f69627832ef.-1 {}] backup.HFileArchiver(360): Archiving compacted files. 2024-11-24T22:15:55,872 DEBUG [StoreCloser-TestAcidGuarantees,,1732486521610.94f3ccf9f8a638fa974e1f69627832ef.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/94f3ccf9f8a638fa974e1f69627832ef/A/090547ea69e9495fb9b887db92c6c3da to hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/archive/data/default/TestAcidGuarantees/94f3ccf9f8a638fa974e1f69627832ef/A/090547ea69e9495fb9b887db92c6c3da 2024-11-24T22:15:55,873 DEBUG [StoreCloser-TestAcidGuarantees,,1732486521610.94f3ccf9f8a638fa974e1f69627832ef.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/94f3ccf9f8a638fa974e1f69627832ef/A/93e608ac185c42b39eac1782462a5302 to hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/archive/data/default/TestAcidGuarantees/94f3ccf9f8a638fa974e1f69627832ef/A/93e608ac185c42b39eac1782462a5302 2024-11-24T22:15:55,874 DEBUG [StoreCloser-TestAcidGuarantees,,1732486521610.94f3ccf9f8a638fa974e1f69627832ef.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/94f3ccf9f8a638fa974e1f69627832ef/A/57308f1fd93a41fa954bec339b0cb607 to hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/archive/data/default/TestAcidGuarantees/94f3ccf9f8a638fa974e1f69627832ef/A/57308f1fd93a41fa954bec339b0cb607 2024-11-24T22:15:55,874 DEBUG [StoreCloser-TestAcidGuarantees,,1732486521610.94f3ccf9f8a638fa974e1f69627832ef.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/94f3ccf9f8a638fa974e1f69627832ef/A/8d09d43826484065b9293366a1476cb5 to hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/archive/data/default/TestAcidGuarantees/94f3ccf9f8a638fa974e1f69627832ef/A/8d09d43826484065b9293366a1476cb5 2024-11-24T22:15:55,875 DEBUG [StoreCloser-TestAcidGuarantees,,1732486521610.94f3ccf9f8a638fa974e1f69627832ef.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/94f3ccf9f8a638fa974e1f69627832ef/A/19c3184bc0c8467ea8a35a00f8b1bd84 to hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/archive/data/default/TestAcidGuarantees/94f3ccf9f8a638fa974e1f69627832ef/A/19c3184bc0c8467ea8a35a00f8b1bd84 2024-11-24T22:15:55,876 DEBUG [StoreCloser-TestAcidGuarantees,,1732486521610.94f3ccf9f8a638fa974e1f69627832ef.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/94f3ccf9f8a638fa974e1f69627832ef/A/510a774ece96416f8abbaee4f4c581cd to hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/archive/data/default/TestAcidGuarantees/94f3ccf9f8a638fa974e1f69627832ef/A/510a774ece96416f8abbaee4f4c581cd 2024-11-24T22:15:55,877 DEBUG [StoreCloser-TestAcidGuarantees,,1732486521610.94f3ccf9f8a638fa974e1f69627832ef.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/94f3ccf9f8a638fa974e1f69627832ef/A/0aaf705b0aec416c8bee1e69de45a7d4 to hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/archive/data/default/TestAcidGuarantees/94f3ccf9f8a638fa974e1f69627832ef/A/0aaf705b0aec416c8bee1e69de45a7d4 2024-11-24T22:15:55,878 DEBUG [StoreCloser-TestAcidGuarantees,,1732486521610.94f3ccf9f8a638fa974e1f69627832ef.-1 {}] regionserver.HStore(2317): Moving the files [hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/94f3ccf9f8a638fa974e1f69627832ef/B/3ef534ba6992421fb44a53ef9a12e571, hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/94f3ccf9f8a638fa974e1f69627832ef/B/5cbd879d83374485b483f18f6194ad70, hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/94f3ccf9f8a638fa974e1f69627832ef/B/149583da245f45179d6de44960e3e48e, hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/94f3ccf9f8a638fa974e1f69627832ef/B/154e2a22f31844d198f56aa1a33d20ca, hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/94f3ccf9f8a638fa974e1f69627832ef/B/472d5538cdba45b3ae87ece46ad44a5e, hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/94f3ccf9f8a638fa974e1f69627832ef/B/32aece1bb9144ffb8c84c5135450995b, hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/94f3ccf9f8a638fa974e1f69627832ef/B/a46e3bf0d2864cdf9c09d1df707b7fa9] to archive 2024-11-24T22:15:55,879 DEBUG [StoreCloser-TestAcidGuarantees,,1732486521610.94f3ccf9f8a638fa974e1f69627832ef.-1 {}] backup.HFileArchiver(360): Archiving compacted files. 2024-11-24T22:15:55,880 DEBUG [StoreCloser-TestAcidGuarantees,,1732486521610.94f3ccf9f8a638fa974e1f69627832ef.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/94f3ccf9f8a638fa974e1f69627832ef/B/3ef534ba6992421fb44a53ef9a12e571 to hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/archive/data/default/TestAcidGuarantees/94f3ccf9f8a638fa974e1f69627832ef/B/3ef534ba6992421fb44a53ef9a12e571 2024-11-24T22:15:55,880 DEBUG [StoreCloser-TestAcidGuarantees,,1732486521610.94f3ccf9f8a638fa974e1f69627832ef.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/94f3ccf9f8a638fa974e1f69627832ef/B/5cbd879d83374485b483f18f6194ad70 to hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/archive/data/default/TestAcidGuarantees/94f3ccf9f8a638fa974e1f69627832ef/B/5cbd879d83374485b483f18f6194ad70 2024-11-24T22:15:55,881 DEBUG [StoreCloser-TestAcidGuarantees,,1732486521610.94f3ccf9f8a638fa974e1f69627832ef.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/94f3ccf9f8a638fa974e1f69627832ef/B/149583da245f45179d6de44960e3e48e to hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/archive/data/default/TestAcidGuarantees/94f3ccf9f8a638fa974e1f69627832ef/B/149583da245f45179d6de44960e3e48e 2024-11-24T22:15:55,882 DEBUG [StoreCloser-TestAcidGuarantees,,1732486521610.94f3ccf9f8a638fa974e1f69627832ef.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/94f3ccf9f8a638fa974e1f69627832ef/B/154e2a22f31844d198f56aa1a33d20ca to hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/archive/data/default/TestAcidGuarantees/94f3ccf9f8a638fa974e1f69627832ef/B/154e2a22f31844d198f56aa1a33d20ca 2024-11-24T22:15:55,883 DEBUG [StoreCloser-TestAcidGuarantees,,1732486521610.94f3ccf9f8a638fa974e1f69627832ef.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/94f3ccf9f8a638fa974e1f69627832ef/B/472d5538cdba45b3ae87ece46ad44a5e to hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/archive/data/default/TestAcidGuarantees/94f3ccf9f8a638fa974e1f69627832ef/B/472d5538cdba45b3ae87ece46ad44a5e 2024-11-24T22:15:55,884 DEBUG [StoreCloser-TestAcidGuarantees,,1732486521610.94f3ccf9f8a638fa974e1f69627832ef.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/94f3ccf9f8a638fa974e1f69627832ef/B/32aece1bb9144ffb8c84c5135450995b to hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/archive/data/default/TestAcidGuarantees/94f3ccf9f8a638fa974e1f69627832ef/B/32aece1bb9144ffb8c84c5135450995b 2024-11-24T22:15:55,884 DEBUG [StoreCloser-TestAcidGuarantees,,1732486521610.94f3ccf9f8a638fa974e1f69627832ef.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/94f3ccf9f8a638fa974e1f69627832ef/B/a46e3bf0d2864cdf9c09d1df707b7fa9 to hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/archive/data/default/TestAcidGuarantees/94f3ccf9f8a638fa974e1f69627832ef/B/a46e3bf0d2864cdf9c09d1df707b7fa9 2024-11-24T22:15:55,885 DEBUG [StoreCloser-TestAcidGuarantees,,1732486521610.94f3ccf9f8a638fa974e1f69627832ef.-1 {}] regionserver.HStore(2317): Moving the files [hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/94f3ccf9f8a638fa974e1f69627832ef/C/33635f7b1a774316b05468b61e43d862, hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/94f3ccf9f8a638fa974e1f69627832ef/C/25ca7dfcdf7c4069ba937e2d7f4fc7e7, hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/94f3ccf9f8a638fa974e1f69627832ef/C/62d1d497a09e4e0fb4316beebe854732, hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/94f3ccf9f8a638fa974e1f69627832ef/C/73ea5ac0072e48f180abd4ef66e7d3fe, hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/94f3ccf9f8a638fa974e1f69627832ef/C/4920a0a11aed4fdfbbaa49f3afcc84b1, hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/94f3ccf9f8a638fa974e1f69627832ef/C/d8e07a696b374dd38a0665746ee004b3, hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/94f3ccf9f8a638fa974e1f69627832ef/C/d953442a120044119d80665c2d2c97c0] to archive 2024-11-24T22:15:55,886 DEBUG [StoreCloser-TestAcidGuarantees,,1732486521610.94f3ccf9f8a638fa974e1f69627832ef.-1 {}] backup.HFileArchiver(360): Archiving compacted files. 2024-11-24T22:15:55,887 DEBUG [StoreCloser-TestAcidGuarantees,,1732486521610.94f3ccf9f8a638fa974e1f69627832ef.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/94f3ccf9f8a638fa974e1f69627832ef/C/33635f7b1a774316b05468b61e43d862 to hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/archive/data/default/TestAcidGuarantees/94f3ccf9f8a638fa974e1f69627832ef/C/33635f7b1a774316b05468b61e43d862 2024-11-24T22:15:55,888 DEBUG [StoreCloser-TestAcidGuarantees,,1732486521610.94f3ccf9f8a638fa974e1f69627832ef.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/94f3ccf9f8a638fa974e1f69627832ef/C/25ca7dfcdf7c4069ba937e2d7f4fc7e7 to hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/archive/data/default/TestAcidGuarantees/94f3ccf9f8a638fa974e1f69627832ef/C/25ca7dfcdf7c4069ba937e2d7f4fc7e7 2024-11-24T22:15:55,888 DEBUG [StoreCloser-TestAcidGuarantees,,1732486521610.94f3ccf9f8a638fa974e1f69627832ef.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/94f3ccf9f8a638fa974e1f69627832ef/C/62d1d497a09e4e0fb4316beebe854732 to hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/archive/data/default/TestAcidGuarantees/94f3ccf9f8a638fa974e1f69627832ef/C/62d1d497a09e4e0fb4316beebe854732 2024-11-24T22:15:55,889 DEBUG [StoreCloser-TestAcidGuarantees,,1732486521610.94f3ccf9f8a638fa974e1f69627832ef.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/94f3ccf9f8a638fa974e1f69627832ef/C/73ea5ac0072e48f180abd4ef66e7d3fe to hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/archive/data/default/TestAcidGuarantees/94f3ccf9f8a638fa974e1f69627832ef/C/73ea5ac0072e48f180abd4ef66e7d3fe 2024-11-24T22:15:55,890 DEBUG [StoreCloser-TestAcidGuarantees,,1732486521610.94f3ccf9f8a638fa974e1f69627832ef.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/94f3ccf9f8a638fa974e1f69627832ef/C/4920a0a11aed4fdfbbaa49f3afcc84b1 to hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/archive/data/default/TestAcidGuarantees/94f3ccf9f8a638fa974e1f69627832ef/C/4920a0a11aed4fdfbbaa49f3afcc84b1 2024-11-24T22:15:55,891 DEBUG [StoreCloser-TestAcidGuarantees,,1732486521610.94f3ccf9f8a638fa974e1f69627832ef.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/94f3ccf9f8a638fa974e1f69627832ef/C/d8e07a696b374dd38a0665746ee004b3 to hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/archive/data/default/TestAcidGuarantees/94f3ccf9f8a638fa974e1f69627832ef/C/d8e07a696b374dd38a0665746ee004b3 2024-11-24T22:15:55,891 DEBUG [StoreCloser-TestAcidGuarantees,,1732486521610.94f3ccf9f8a638fa974e1f69627832ef.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/94f3ccf9f8a638fa974e1f69627832ef/C/d953442a120044119d80665c2d2c97c0 to hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/archive/data/default/TestAcidGuarantees/94f3ccf9f8a638fa974e1f69627832ef/C/d953442a120044119d80665c2d2c97c0 2024-11-24T22:15:55,895 DEBUG [RS_CLOSE_REGION-regionserver/bd6592c37bb2:0-0 {event_type=M_RS_CLOSE_REGION, pid=116}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/94f3ccf9f8a638fa974e1f69627832ef/recovered.edits/324.seqid, newMaxSeqId=324, maxSeqId=4 2024-11-24T22:15:55,895 INFO [RS_CLOSE_REGION-regionserver/bd6592c37bb2:0-0 {event_type=M_RS_CLOSE_REGION, pid=116}] regionserver.HRegion(1973): Closed TestAcidGuarantees,,1732486521610.94f3ccf9f8a638fa974e1f69627832ef. 2024-11-24T22:15:55,895 DEBUG [RS_CLOSE_REGION-regionserver/bd6592c37bb2:0-0 {event_type=M_RS_CLOSE_REGION, pid=116}] regionserver.HRegion(1676): Region close journal for 94f3ccf9f8a638fa974e1f69627832ef: Waiting for close lock at 1732486554214Running coprocessor pre-close hooks at 1732486554214Disabling compacts and flushes for region at 1732486554214Disabling writes for close at 1732486554214Obtaining lock to block concurrent updates at 1732486554214Preparing flush snapshotting stores in 94f3ccf9f8a638fa974e1f69627832ef at 1732486554214Finished memstore snapshotting TestAcidGuarantees,,1732486521610.94f3ccf9f8a638fa974e1f69627832ef., syncing WAL and waiting on mvcc, flushsize=dataSize=13740, getHeapSize=41520, getOffHeapSize=0, getCellsCount=300 at 1732486554215 (+1 ms)Flushing stores of TestAcidGuarantees,,1732486521610.94f3ccf9f8a638fa974e1f69627832ef. at 1732486554215Flushing 94f3ccf9f8a638fa974e1f69627832ef/A: creating writer at 1732486554215Flushing mob file 94f3ccf9f8a638fa974e1f69627832ef/A: appending metadata at 1732486554223 (+8 ms)Flushing mob file 94f3ccf9f8a638fa974e1f69627832ef/A: closing flushed file at 1732486554223Flushing 94f3ccf9f8a638fa974e1f69627832ef/A: appending metadata at 1732486554632 (+409 ms)Flushing 94f3ccf9f8a638fa974e1f69627832ef/A: closing flushed file at 1732486554632Flushing 94f3ccf9f8a638fa974e1f69627832ef/B: creating writer at 1732486555039 (+407 ms)Flushing 94f3ccf9f8a638fa974e1f69627832ef/B: appending metadata at 1732486555041 (+2 ms)Flushing 94f3ccf9f8a638fa974e1f69627832ef/B: closing flushed file at 1732486555041Flushing 94f3ccf9f8a638fa974e1f69627832ef/C: creating writer at 1732486555449 (+408 ms)Flushing 94f3ccf9f8a638fa974e1f69627832ef/C: appending metadata at 1732486555452 (+3 ms)Flushing 94f3ccf9f8a638fa974e1f69627832ef/C: closing flushed file at 1732486555452Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@6234817e: reopening flushed file at 1732486555859 (+407 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@47742cab: reopening flushed file at 1732486555863 (+4 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@54d34fe1: reopening flushed file at 1732486555866 (+3 ms)Finished flush of dataSize ~13.42 KB/13740, heapSize ~40.55 KB/41520, currentSize=0 B/0 for 94f3ccf9f8a638fa974e1f69627832ef in 1656ms, sequenceid=321, compaction requested=false at 1732486555870 (+4 ms)Writing region close event to WAL at 1732486555892 (+22 ms)Running coprocessor post-close hooks at 1732486555895 (+3 ms)Closed at 1732486555895 2024-11-24T22:15:55,897 INFO [RS_CLOSE_REGION-regionserver/bd6592c37bb2:0-0 {event_type=M_RS_CLOSE_REGION, pid=116}] handler.UnassignRegionHandler(157): Closed 94f3ccf9f8a638fa974e1f69627832ef 2024-11-24T22:15:55,897 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=115 updating hbase:meta row=94f3ccf9f8a638fa974e1f69627832ef, regionState=CLOSED 2024-11-24T22:15:55,899 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=116, ppid=115, state=RUNNABLE, hasLock=false; CloseRegionProcedure 94f3ccf9f8a638fa974e1f69627832ef, server=bd6592c37bb2,40087,1732486423713 because future has completed 2024-11-24T22:15:55,901 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=116, resume processing ppid=115 2024-11-24T22:15:55,901 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=116, ppid=115, state=SUCCESS, hasLock=false; CloseRegionProcedure 94f3ccf9f8a638fa974e1f69627832ef, server=bd6592c37bb2,40087,1732486423713 in 1.8370 sec 2024-11-24T22:15:55,902 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=115, resume processing ppid=114 2024-11-24T22:15:55,902 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=115, ppid=114, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=94f3ccf9f8a638fa974e1f69627832ef, UNASSIGN in 1.8410 sec 2024-11-24T22:15:55,904 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=114, resume processing ppid=113 2024-11-24T22:15:55,904 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=114, ppid=113, state=SUCCESS, hasLock=false; CloseTableRegionsProcedure table=TestAcidGuarantees in 1.8450 sec 2024-11-24T22:15:55,905 DEBUG [PEWorker-3 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1732486555905"}]},"ts":"1732486555905"} 2024-11-24T22:15:55,906 INFO [PEWorker-3 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=DISABLED in hbase:meta 2024-11-24T22:15:55,906 INFO [PEWorker-3 {}] procedure.DisableTableProcedure(296): Set TestAcidGuarantees to state=DISABLED 2024-11-24T22:15:55,908 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=113, state=SUCCESS, hasLock=false; DisableTableProcedure table=TestAcidGuarantees in 1.8530 sec 2024-11-24T22:15:56,201 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37701 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=113 2024-11-24T22:15:56,201 INFO [RPCClient-NioEventLoopGroup-4-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: DISABLE, Table Name: default:TestAcidGuarantees completed 2024-11-24T22:15:56,202 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37701 {}] master.HMaster$5(2570): Client=jenkins//172.17.0.2 delete TestAcidGuarantees 2024-11-24T22:15:56,203 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37701 {}] procedure2.ProcedureExecutor(1139): Stored pid=117, state=RUNNABLE:DELETE_TABLE_PRE_OPERATION, hasLock=false; DeleteTableProcedure table=TestAcidGuarantees 2024-11-24T22:15:56,204 DEBUG [PEWorker-1 {}] procedure.DeleteTableProcedure(103): Waiting for RIT for pid=117, state=RUNNABLE:DELETE_TABLE_PRE_OPERATION, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-24T22:15:56,204 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37701 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=117 2024-11-24T22:15:56,204 DEBUG [PEWorker-1 {}] procedure.DeleteTableProcedure(115): Deleting regions from filesystem for pid=117, state=RUNNABLE:DELETE_TABLE_CLEAR_FS_LAYOUT, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-24T22:15:56,207 DEBUG [HFileArchiver-4 {}] backup.HFileArchiver(131): ARCHIVING hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/94f3ccf9f8a638fa974e1f69627832ef 2024-11-24T22:15:56,208 DEBUG [HFileArchiver-4 {}] backup.HFileArchiver(159): Archiving [FileablePath, hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/94f3ccf9f8a638fa974e1f69627832ef/A, FileablePath, hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/94f3ccf9f8a638fa974e1f69627832ef/B, FileablePath, hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/94f3ccf9f8a638fa974e1f69627832ef/C, FileablePath, hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/94f3ccf9f8a638fa974e1f69627832ef/recovered.edits] 2024-11-24T22:15:56,211 DEBUG [HFileArchiver-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/94f3ccf9f8a638fa974e1f69627832ef/A/d968deefbf564bcc9d4adf3de5e8cd45 to hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/archive/data/default/TestAcidGuarantees/94f3ccf9f8a638fa974e1f69627832ef/A/d968deefbf564bcc9d4adf3de5e8cd45 2024-11-24T22:15:56,212 DEBUG [HFileArchiver-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/94f3ccf9f8a638fa974e1f69627832ef/A/de25378ccb7a422d967b6c53762341c8 to hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/archive/data/default/TestAcidGuarantees/94f3ccf9f8a638fa974e1f69627832ef/A/de25378ccb7a422d967b6c53762341c8 2024-11-24T22:15:56,214 DEBUG [HFileArchiver-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/94f3ccf9f8a638fa974e1f69627832ef/B/82a4e0bfd1514c5aa8c9fce35ee180f7 to hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/archive/data/default/TestAcidGuarantees/94f3ccf9f8a638fa974e1f69627832ef/B/82a4e0bfd1514c5aa8c9fce35ee180f7 2024-11-24T22:15:56,215 DEBUG [HFileArchiver-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/94f3ccf9f8a638fa974e1f69627832ef/B/836bd1834e214114983e4a2790195396 to hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/archive/data/default/TestAcidGuarantees/94f3ccf9f8a638fa974e1f69627832ef/B/836bd1834e214114983e4a2790195396 2024-11-24T22:15:56,216 DEBUG [HFileArchiver-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/94f3ccf9f8a638fa974e1f69627832ef/C/37f2846949b4429bb215ef889bd8a7f0 to hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/archive/data/default/TestAcidGuarantees/94f3ccf9f8a638fa974e1f69627832ef/C/37f2846949b4429bb215ef889bd8a7f0 2024-11-24T22:15:56,217 DEBUG [HFileArchiver-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/94f3ccf9f8a638fa974e1f69627832ef/C/dabbccd5526b4836aa77269cf112f130 to hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/archive/data/default/TestAcidGuarantees/94f3ccf9f8a638fa974e1f69627832ef/C/dabbccd5526b4836aa77269cf112f130 2024-11-24T22:15:56,219 DEBUG [HFileArchiver-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/94f3ccf9f8a638fa974e1f69627832ef/recovered.edits/324.seqid to hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/archive/data/default/TestAcidGuarantees/94f3ccf9f8a638fa974e1f69627832ef/recovered.edits/324.seqid 2024-11-24T22:15:56,219 DEBUG [HFileArchiver-4 {}] backup.HFileArchiver(610): Deleted hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/94f3ccf9f8a638fa974e1f69627832ef 2024-11-24T22:15:56,219 DEBUG [PEWorker-1 {}] procedure.DeleteTableProcedure(313): Archived TestAcidGuarantees regions 2024-11-24T22:15:56,219 DEBUG [PEWorker-1 {}] backup.HFileArchiver(131): ARCHIVING hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3 2024-11-24T22:15:56,220 DEBUG [PEWorker-1 {}] backup.HFileArchiver(159): Archiving [FileablePath, hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A] 2024-11-24T22:15:56,222 DEBUG [PEWorker-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241124067acc14c06a447ea8f164340f4cdd56_94f3ccf9f8a638fa974e1f69627832ef to hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241124067acc14c06a447ea8f164340f4cdd56_94f3ccf9f8a638fa974e1f69627832ef 2024-11-24T22:15:56,223 DEBUG [PEWorker-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024112412ed540505d4421b925a2bae62d50a4e_94f3ccf9f8a638fa974e1f69627832ef to hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024112412ed540505d4421b925a2bae62d50a4e_94f3ccf9f8a638fa974e1f69627832ef 2024-11-24T22:15:56,224 DEBUG [PEWorker-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241124250a21fc5d534c768d85cee87472e326_94f3ccf9f8a638fa974e1f69627832ef to hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241124250a21fc5d534c768d85cee87472e326_94f3ccf9f8a638fa974e1f69627832ef 2024-11-24T22:15:56,224 DEBUG [PEWorker-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411242bd23b0eed0e4f898ce86e11ba5d59a0_94f3ccf9f8a638fa974e1f69627832ef to hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411242bd23b0eed0e4f898ce86e11ba5d59a0_94f3ccf9f8a638fa974e1f69627832ef 2024-11-24T22:15:56,225 DEBUG [PEWorker-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024112433497985dd284519a218a410b01c5320_94f3ccf9f8a638fa974e1f69627832ef to hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024112433497985dd284519a218a410b01c5320_94f3ccf9f8a638fa974e1f69627832ef 2024-11-24T22:15:56,226 DEBUG [PEWorker-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411245775428d17684f829ea4eb940d9c82dd_94f3ccf9f8a638fa974e1f69627832ef to hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411245775428d17684f829ea4eb940d9c82dd_94f3ccf9f8a638fa974e1f69627832ef 2024-11-24T22:15:56,227 DEBUG [PEWorker-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411247f8a7e400f40437488012456cbc688bf_94f3ccf9f8a638fa974e1f69627832ef to hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411247f8a7e400f40437488012456cbc688bf_94f3ccf9f8a638fa974e1f69627832ef 2024-11-24T22:15:56,227 DEBUG [PEWorker-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411247fa1ae1fd2c54228a8ce550e28aa4a3d_94f3ccf9f8a638fa974e1f69627832ef to hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411247fa1ae1fd2c54228a8ce550e28aa4a3d_94f3ccf9f8a638fa974e1f69627832ef 2024-11-24T22:15:56,228 DEBUG [PEWorker-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024112483afbdca7932409c9c912608cdcdfbb3_94f3ccf9f8a638fa974e1f69627832ef to hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024112483afbdca7932409c9c912608cdcdfbb3_94f3ccf9f8a638fa974e1f69627832ef 2024-11-24T22:15:56,229 DEBUG [PEWorker-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024112488d65ced9a884882a2ed516a7cb18106_94f3ccf9f8a638fa974e1f69627832ef to hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024112488d65ced9a884882a2ed516a7cb18106_94f3ccf9f8a638fa974e1f69627832ef 2024-11-24T22:15:56,229 DEBUG [PEWorker-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241124939393c18b524edc9db85217878b0130_94f3ccf9f8a638fa974e1f69627832ef to hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241124939393c18b524edc9db85217878b0130_94f3ccf9f8a638fa974e1f69627832ef 2024-11-24T22:15:56,230 DEBUG [PEWorker-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411249b392706f214429eaa46998773c08a09_94f3ccf9f8a638fa974e1f69627832ef to hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411249b392706f214429eaa46998773c08a09_94f3ccf9f8a638fa974e1f69627832ef 2024-11-24T22:15:56,231 DEBUG [PEWorker-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241124ad2a51c8febb43ad978a48d99b3b0548_94f3ccf9f8a638fa974e1f69627832ef to hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241124ad2a51c8febb43ad978a48d99b3b0548_94f3ccf9f8a638fa974e1f69627832ef 2024-11-24T22:15:56,232 DEBUG [PEWorker-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241124ba52deec5a5a4309abd261c1086afe25_94f3ccf9f8a638fa974e1f69627832ef to hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241124ba52deec5a5a4309abd261c1086afe25_94f3ccf9f8a638fa974e1f69627832ef 2024-11-24T22:15:56,233 DEBUG [PEWorker-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241124be89bcf0c9464cd7a6332888a52d26a0_94f3ccf9f8a638fa974e1f69627832ef to hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241124be89bcf0c9464cd7a6332888a52d26a0_94f3ccf9f8a638fa974e1f69627832ef 2024-11-24T22:15:56,234 DEBUG [PEWorker-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241124cc8a4013fb4f4d0780b44c16d7bd24b5_94f3ccf9f8a638fa974e1f69627832ef to hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241124cc8a4013fb4f4d0780b44c16d7bd24b5_94f3ccf9f8a638fa974e1f69627832ef 2024-11-24T22:15:56,234 DEBUG [PEWorker-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241124d09a5d75e35d448e8a2bc00040a143ab_94f3ccf9f8a638fa974e1f69627832ef to hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241124d09a5d75e35d448e8a2bc00040a143ab_94f3ccf9f8a638fa974e1f69627832ef 2024-11-24T22:15:56,235 DEBUG [PEWorker-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241124e4beb5734e154df990570008741226e0_94f3ccf9f8a638fa974e1f69627832ef to hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241124e4beb5734e154df990570008741226e0_94f3ccf9f8a638fa974e1f69627832ef 2024-11-24T22:15:56,236 DEBUG [PEWorker-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241124fb1ff857056d430e8a2a87ac743e865a_94f3ccf9f8a638fa974e1f69627832ef to hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241124fb1ff857056d430e8a2a87ac743e865a_94f3ccf9f8a638fa974e1f69627832ef 2024-11-24T22:15:56,236 DEBUG [PEWorker-1 {}] backup.HFileArchiver(610): Deleted hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3 2024-11-24T22:15:56,238 DEBUG [PEWorker-1 {}] procedure.DeleteTableProcedure(120): Deleting regions from META for pid=117, state=RUNNABLE:DELETE_TABLE_REMOVE_FROM_META, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-24T22:15:56,240 WARN [PEWorker-1 {}] procedure.DeleteTableProcedure(370): Deleting some vestigial 1 rows of TestAcidGuarantees from hbase:meta 2024-11-24T22:15:56,242 DEBUG [PEWorker-1 {}] procedure.DeleteTableProcedure(407): Removing 'TestAcidGuarantees' descriptor. 2024-11-24T22:15:56,242 DEBUG [PEWorker-1 {}] procedure.DeleteTableProcedure(126): Deleting assignment state for pid=117, state=RUNNABLE:DELETE_TABLE_UNASSIGN_REGIONS, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-24T22:15:56,242 DEBUG [PEWorker-1 {}] procedure.DeleteTableProcedure(397): Removing 'TestAcidGuarantees' from region states. 2024-11-24T22:15:56,243 DEBUG [PEWorker-1 {}] assignment.RegionStateStore(723): Delete {"totalColumns":1,"row":"TestAcidGuarantees,,1732486521610.94f3ccf9f8a638fa974e1f69627832ef.","families":{"info":[{"qualifier":"","vlen":0,"tag":[],"timestamp":"1732486556242"}]},"ts":"9223372036854775807"} 2024-11-24T22:15:56,244 INFO [PEWorker-1 {}] assignment.RegionStateStore(562): Deleted 1 regions from META 2024-11-24T22:15:56,244 DEBUG [PEWorker-1 {}] assignment.RegionStateStore(563): Deleted regions: [{ENCODED => 94f3ccf9f8a638fa974e1f69627832ef, NAME => 'TestAcidGuarantees,,1732486521610.94f3ccf9f8a638fa974e1f69627832ef.', STARTKEY => '', ENDKEY => ''}] 2024-11-24T22:15:56,244 DEBUG [PEWorker-1 {}] procedure.DeleteTableProcedure(401): Marking 'TestAcidGuarantees' as deleted. 2024-11-24T22:15:56,244 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Delete {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":0,"tag":[],"timestamp":"1732486556244"}]},"ts":"9223372036854775807"} 2024-11-24T22:15:56,246 INFO [PEWorker-1 {}] hbase.MetaTableAccessor(867): Deleted table TestAcidGuarantees state from META 2024-11-24T22:15:56,246 DEBUG [PEWorker-1 {}] procedure.DeleteTableProcedure(133): Finished pid=117, state=RUNNABLE:DELETE_TABLE_POST_OPERATION, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-24T22:15:56,247 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=117, state=SUCCESS, hasLock=false; DeleteTableProcedure table=TestAcidGuarantees in 44 msec 2024-11-24T22:15:56,311 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37701 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=117 2024-11-24T22:15:56,311 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocator(219): Clear meta cache for TestAcidGuarantees 2024-11-24T22:15:56,313 INFO [RPCClient-NioEventLoopGroup-4-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: DELETE, Table Name: default:TestAcidGuarantees completed 2024-11-24T22:15:56,320 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: TestAcidGuaranteesWithEagerPolicy#testMobScanAtomicity Thread=249 (was 248) - Thread LEAK? -, OpenFileDescriptor=449 (was 455), MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=310 (was 374), ProcessCount=11 (was 11), AvailableMemoryMB=5852 (was 5959) 2024-11-24T22:15:56,328 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: TestAcidGuaranteesWithEagerPolicy#testScanAtomicity Thread=249, OpenFileDescriptor=449, MaxFileDescriptor=1048576, SystemLoadAverage=310, ProcessCount=11, AvailableMemoryMB=5852 2024-11-24T22:15:56,330 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37701 {}] util.TableDescriptorChecker(321): MEMSTORE_FLUSHSIZE for table descriptor or "hbase.hregion.memstore.flush.size" (131072) is too small, which might cause very frequent flushing. 2024-11-24T22:15:56,330 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37701 {}] master.HMaster$4(2454): Client=jenkins//172.17.0.2 create 'TestAcidGuarantees', {TABLE_ATTRIBUTES => {METADATA => {'hbase.hregion.compacting.memstore.type' => 'EAGER', 'hbase.hregion.memstore.mslab.enabled' => 'false', 'hbase.memstore.inmemoryflush.threshold.factor' => '0.9'}}}, {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-11-24T22:15:56,331 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37701 {}] procedure2.ProcedureExecutor(1139): Stored pid=118, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=false; CreateTableProcedure table=TestAcidGuarantees 2024-11-24T22:15:56,332 INFO [PEWorker-2 {}] procedure.CreateTableProcedure(89): pid=118, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_PRE_OPERATION 2024-11-24T22:15:56,332 DEBUG [PEWorker-2 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T22:15:56,332 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37701 {}] master.MasterRpcServices(787): Client=jenkins//172.17.0.2 procedure request for creating table: namespace: "default" qualifier: "TestAcidGuarantees" procId is: 118 2024-11-24T22:15:56,332 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37701 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=118 2024-11-24T22:15:56,332 INFO [PEWorker-2 {}] procedure.CreateTableProcedure(89): pid=118, state=RUNNABLE:CREATE_TABLE_WRITE_FS_LAYOUT, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_WRITE_FS_LAYOUT 2024-11-24T22:15:56,337 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45929 is added to blk_1073742276_1452 (size=1061) 2024-11-24T22:15:56,441 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37701 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=118 2024-11-24T22:15:56,650 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37701 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=118 2024-11-24T22:15:56,739 INFO [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(7572): creating {ENCODED => 744bbec6934be304e01b009be760323d, NAME => 'TestAcidGuarantees,,1732486556329.744bbec6934be304e01b009be760323d.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='TestAcidGuarantees', {TABLE_ATTRIBUTES => {METADATA => {'hbase.hregion.compacting.memstore.type' => 'EAGER', 'hbase.hregion.memstore.mslab.enabled' => 'false', 'hbase.memstore.inmemoryflush.threshold.factor' => '0.9', 'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc 2024-11-24T22:15:56,743 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45929 is added to blk_1073742277_1453 (size=53) 2024-11-24T22:15:56,961 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37701 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=118 2024-11-24T22:15:57,144 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(898): Instantiated TestAcidGuarantees,,1732486556329.744bbec6934be304e01b009be760323d.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-24T22:15:57,144 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1722): Closing 744bbec6934be304e01b009be760323d, disabling compactions & flushes 2024-11-24T22:15:57,144 INFO [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1755): Closing region TestAcidGuarantees,,1732486556329.744bbec6934be304e01b009be760323d. 2024-11-24T22:15:57,144 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1776): Time limited wait for close lock on TestAcidGuarantees,,1732486556329.744bbec6934be304e01b009be760323d. 2024-11-24T22:15:57,144 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1843): Acquired close lock on TestAcidGuarantees,,1732486556329.744bbec6934be304e01b009be760323d. after waiting 0 ms 2024-11-24T22:15:57,144 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1853): Updates disabled for region TestAcidGuarantees,,1732486556329.744bbec6934be304e01b009be760323d. 2024-11-24T22:15:57,144 INFO [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1973): Closed TestAcidGuarantees,,1732486556329.744bbec6934be304e01b009be760323d. 2024-11-24T22:15:57,144 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1676): Region close journal for 744bbec6934be304e01b009be760323d: Waiting for close lock at 1732486557144Disabling compacts and flushes for region at 1732486557144Disabling writes for close at 1732486557144Writing region close event to WAL at 1732486557144Closed at 1732486557144 2024-11-24T22:15:57,145 INFO [PEWorker-2 {}] procedure.CreateTableProcedure(89): pid=118, state=RUNNABLE:CREATE_TABLE_ADD_TO_META, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_ADD_TO_META 2024-11-24T22:15:57,145 DEBUG [PEWorker-2 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":2,"row":"TestAcidGuarantees,,1732486556329.744bbec6934be304e01b009be760323d.","families":{"info":[{"qualifier":"regioninfo","vlen":52,"tag":[],"timestamp":"1732486557145"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1732486557145"}]},"ts":"1732486557145"} 2024-11-24T22:15:57,147 INFO [PEWorker-2 {}] hbase.MetaTableAccessor(832): Added 1 regions to meta. 2024-11-24T22:15:57,147 INFO [PEWorker-2 {}] procedure.CreateTableProcedure(89): pid=118, state=RUNNABLE:CREATE_TABLE_ASSIGN_REGIONS, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_ASSIGN_REGIONS 2024-11-24T22:15:57,147 DEBUG [PEWorker-2 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1732486557147"}]},"ts":"1732486557147"} 2024-11-24T22:15:57,148 INFO [PEWorker-2 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=ENABLING in hbase:meta 2024-11-24T22:15:57,148 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=119, ppid=118, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=744bbec6934be304e01b009be760323d, ASSIGN}] 2024-11-24T22:15:57,149 INFO [PEWorker-4 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=119, ppid=118, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=744bbec6934be304e01b009be760323d, ASSIGN 2024-11-24T22:15:57,150 INFO [PEWorker-4 {}] assignment.TransitRegionStateProcedure(269): Starting pid=119, ppid=118, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=744bbec6934be304e01b009be760323d, ASSIGN; state=OFFLINE, location=bd6592c37bb2,40087,1732486423713; forceNewPlan=false, retain=false 2024-11-24T22:15:57,301 INFO [PEWorker-5 {}] assignment.RegionStateStore(223): pid=119 updating hbase:meta row=744bbec6934be304e01b009be760323d, regionState=OPENING, regionLocation=bd6592c37bb2,40087,1732486423713 2024-11-24T22:15:57,302 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=119, ppid=118, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=744bbec6934be304e01b009be760323d, ASSIGN because future has completed 2024-11-24T22:15:57,302 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=120, ppid=119, state=RUNNABLE, hasLock=false; OpenRegionProcedure 744bbec6934be304e01b009be760323d, server=bd6592c37bb2,40087,1732486423713}] 2024-11-24T22:15:57,456 INFO [RS_OPEN_REGION-regionserver/bd6592c37bb2:0-0 {event_type=M_RS_OPEN_REGION, pid=120}] handler.AssignRegionHandler(132): Open TestAcidGuarantees,,1732486556329.744bbec6934be304e01b009be760323d. 2024-11-24T22:15:57,456 DEBUG [RS_OPEN_REGION-regionserver/bd6592c37bb2:0-0 {event_type=M_RS_OPEN_REGION, pid=120}] regionserver.HRegion(7752): Opening region: {ENCODED => 744bbec6934be304e01b009be760323d, NAME => 'TestAcidGuarantees,,1732486556329.744bbec6934be304e01b009be760323d.', STARTKEY => '', ENDKEY => ''} 2024-11-24T22:15:57,457 DEBUG [RS_OPEN_REGION-regionserver/bd6592c37bb2:0-0 {event_type=M_RS_OPEN_REGION, pid=120}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table TestAcidGuarantees 744bbec6934be304e01b009be760323d 2024-11-24T22:15:57,457 DEBUG [RS_OPEN_REGION-regionserver/bd6592c37bb2:0-0 {event_type=M_RS_OPEN_REGION, pid=120}] regionserver.HRegion(898): Instantiated TestAcidGuarantees,,1732486556329.744bbec6934be304e01b009be760323d.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-24T22:15:57,457 DEBUG [RS_OPEN_REGION-regionserver/bd6592c37bb2:0-0 {event_type=M_RS_OPEN_REGION, pid=120}] regionserver.HRegion(7794): checking encryption for 744bbec6934be304e01b009be760323d 2024-11-24T22:15:57,457 DEBUG [RS_OPEN_REGION-regionserver/bd6592c37bb2:0-0 {event_type=M_RS_OPEN_REGION, pid=120}] regionserver.HRegion(7797): checking classloading for 744bbec6934be304e01b009be760323d 2024-11-24T22:15:57,458 INFO [StoreOpener-744bbec6934be304e01b009be760323d-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family A of region 744bbec6934be304e01b009be760323d 2024-11-24T22:15:57,459 INFO [StoreOpener-744bbec6934be304e01b009be760323d-1 {}] regionserver.CompactingMemStore(122): Store=A, in-memory flush size threshold=38.40 KB, immutable segments index type=ARRAY_MAP, compactor=EAGER, pipelineThreshold=2, compactionCellMax=10 2024-11-24T22:15:57,460 INFO [StoreOpener-744bbec6934be304e01b009be760323d-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 744bbec6934be304e01b009be760323d columnFamilyName A 2024-11-24T22:15:57,460 DEBUG [StoreOpener-744bbec6934be304e01b009be760323d-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T22:15:57,460 INFO [StoreOpener-744bbec6934be304e01b009be760323d-1 {}] regionserver.HStore(327): Store=744bbec6934be304e01b009be760323d/A, memstore type=CompactingMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-24T22:15:57,460 INFO [StoreOpener-744bbec6934be304e01b009be760323d-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family B of region 744bbec6934be304e01b009be760323d 2024-11-24T22:15:57,461 INFO [StoreOpener-744bbec6934be304e01b009be760323d-1 {}] regionserver.CompactingMemStore(122): Store=B, in-memory flush size threshold=38.40 KB, immutable segments index type=ARRAY_MAP, compactor=EAGER, pipelineThreshold=2, compactionCellMax=10 2024-11-24T22:15:57,461 INFO [StoreOpener-744bbec6934be304e01b009be760323d-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 744bbec6934be304e01b009be760323d columnFamilyName B 2024-11-24T22:15:57,461 DEBUG [StoreOpener-744bbec6934be304e01b009be760323d-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T22:15:57,462 INFO [StoreOpener-744bbec6934be304e01b009be760323d-1 {}] regionserver.HStore(327): Store=744bbec6934be304e01b009be760323d/B, memstore type=CompactingMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-24T22:15:57,462 INFO [StoreOpener-744bbec6934be304e01b009be760323d-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family C of region 744bbec6934be304e01b009be760323d 2024-11-24T22:15:57,462 INFO [StoreOpener-744bbec6934be304e01b009be760323d-1 {}] regionserver.CompactingMemStore(122): Store=C, in-memory flush size threshold=38.40 KB, immutable segments index type=ARRAY_MAP, compactor=EAGER, pipelineThreshold=2, compactionCellMax=10 2024-11-24T22:15:57,462 INFO [StoreOpener-744bbec6934be304e01b009be760323d-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 744bbec6934be304e01b009be760323d columnFamilyName C 2024-11-24T22:15:57,462 DEBUG [StoreOpener-744bbec6934be304e01b009be760323d-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T22:15:57,463 INFO [StoreOpener-744bbec6934be304e01b009be760323d-1 {}] regionserver.HStore(327): Store=744bbec6934be304e01b009be760323d/C, memstore type=CompactingMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-24T22:15:57,463 INFO [RS_OPEN_REGION-regionserver/bd6592c37bb2:0-0 {event_type=M_RS_OPEN_REGION, pid=120}] regionserver.HRegion(1212): Setting FlushNonSloppyStoresFirstPolicy for the region=TestAcidGuarantees,,1732486556329.744bbec6934be304e01b009be760323d. 2024-11-24T22:15:57,463 DEBUG [RS_OPEN_REGION-regionserver/bd6592c37bb2:0-0 {event_type=M_RS_OPEN_REGION, pid=120}] regionserver.HRegion(1038): replaying wal for 744bbec6934be304e01b009be760323d 2024-11-24T22:15:57,463 DEBUG [RS_OPEN_REGION-regionserver/bd6592c37bb2:0-0 {event_type=M_RS_OPEN_REGION, pid=120}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/744bbec6934be304e01b009be760323d 2024-11-24T22:15:57,464 DEBUG [RS_OPEN_REGION-regionserver/bd6592c37bb2:0-0 {event_type=M_RS_OPEN_REGION, pid=120}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/744bbec6934be304e01b009be760323d 2024-11-24T22:15:57,464 DEBUG [RS_OPEN_REGION-regionserver/bd6592c37bb2:0-0 {event_type=M_RS_OPEN_REGION, pid=120}] regionserver.HRegion(1048): stopping wal replay for 744bbec6934be304e01b009be760323d 2024-11-24T22:15:57,464 DEBUG [RS_OPEN_REGION-regionserver/bd6592c37bb2:0-0 {event_type=M_RS_OPEN_REGION, pid=120}] regionserver.HRegion(1060): Cleaning up temporary data for 744bbec6934be304e01b009be760323d 2024-11-24T22:15:57,465 DEBUG [RS_OPEN_REGION-regionserver/bd6592c37bb2:0-0 {event_type=M_RS_OPEN_REGION, pid=120}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table TestAcidGuarantees descriptor;using region.getMemStoreFlushHeapSize/# of families (16.0 M)) instead. 2024-11-24T22:15:57,465 DEBUG [RS_OPEN_REGION-regionserver/bd6592c37bb2:0-0 {event_type=M_RS_OPEN_REGION, pid=120}] regionserver.HRegion(1093): writing seq id for 744bbec6934be304e01b009be760323d 2024-11-24T22:15:57,467 DEBUG [RS_OPEN_REGION-regionserver/bd6592c37bb2:0-0 {event_type=M_RS_OPEN_REGION, pid=120}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/744bbec6934be304e01b009be760323d/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-24T22:15:57,467 INFO [RS_OPEN_REGION-regionserver/bd6592c37bb2:0-0 {event_type=M_RS_OPEN_REGION, pid=120}] regionserver.HRegion(1114): Opened 744bbec6934be304e01b009be760323d; next sequenceid=2; ConstantSizeRegionSplitPolicy{desiredMaxFileSize=64041757, jitterRate=-0.0457034558057785}, FlushLargeStoresPolicy{flushSizeLowerBound=16777216} 2024-11-24T22:15:57,467 DEBUG [RS_OPEN_REGION-regionserver/bd6592c37bb2:0-0 {event_type=M_RS_OPEN_REGION, pid=120}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 744bbec6934be304e01b009be760323d 2024-11-24T22:15:57,468 DEBUG [RS_OPEN_REGION-regionserver/bd6592c37bb2:0-0 {event_type=M_RS_OPEN_REGION, pid=120}] regionserver.HRegion(1006): Region open journal for 744bbec6934be304e01b009be760323d: Running coprocessor pre-open hook at 1732486557458Writing region info on filesystem at 1732486557458Initializing all the Stores at 1732486557458Instantiating store for column family {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732486557458Instantiating store for column family {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732486557458Instantiating store for column family {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732486557458Cleaning up temporary data from old regions at 1732486557464 (+6 ms)Running coprocessor post-open hooks at 1732486557467 (+3 ms)Region opened successfully at 1732486557468 (+1 ms) 2024-11-24T22:15:57,468 INFO [RS_OPEN_REGION-regionserver/bd6592c37bb2:0-0 {event_type=M_RS_OPEN_REGION, pid=120}] regionserver.HRegionServer(2236): Post open deploy tasks for TestAcidGuarantees,,1732486556329.744bbec6934be304e01b009be760323d., pid=120, masterSystemTime=1732486557454 2024-11-24T22:15:57,470 DEBUG [RS_OPEN_REGION-regionserver/bd6592c37bb2:0-0 {event_type=M_RS_OPEN_REGION, pid=120}] regionserver.HRegionServer(2266): Finished post open deploy task for TestAcidGuarantees,,1732486556329.744bbec6934be304e01b009be760323d. 2024-11-24T22:15:57,470 INFO [RS_OPEN_REGION-regionserver/bd6592c37bb2:0-0 {event_type=M_RS_OPEN_REGION, pid=120}] handler.AssignRegionHandler(153): Opened TestAcidGuarantees,,1732486556329.744bbec6934be304e01b009be760323d. 2024-11-24T22:15:57,470 INFO [PEWorker-2 {}] assignment.RegionStateStore(223): pid=119 updating hbase:meta row=744bbec6934be304e01b009be760323d, regionState=OPEN, openSeqNum=2, regionLocation=bd6592c37bb2,40087,1732486423713 2024-11-24T22:15:57,471 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37701 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=118 2024-11-24T22:15:57,471 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=120, ppid=119, state=RUNNABLE, hasLock=false; OpenRegionProcedure 744bbec6934be304e01b009be760323d, server=bd6592c37bb2,40087,1732486423713 because future has completed 2024-11-24T22:15:57,473 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=120, resume processing ppid=119 2024-11-24T22:15:57,473 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=120, ppid=119, state=SUCCESS, hasLock=false; OpenRegionProcedure 744bbec6934be304e01b009be760323d, server=bd6592c37bb2,40087,1732486423713 in 170 msec 2024-11-24T22:15:57,475 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=119, resume processing ppid=118 2024-11-24T22:15:57,475 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=119, ppid=118, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=744bbec6934be304e01b009be760323d, ASSIGN in 325 msec 2024-11-24T22:15:57,475 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=118, state=RUNNABLE:CREATE_TABLE_UPDATE_DESC_CACHE, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_UPDATE_DESC_CACHE 2024-11-24T22:15:57,476 DEBUG [PEWorker-3 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1732486557476"}]},"ts":"1732486557476"} 2024-11-24T22:15:57,477 INFO [PEWorker-3 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=ENABLED in hbase:meta 2024-11-24T22:15:57,477 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=118, state=RUNNABLE:CREATE_TABLE_POST_OPERATION, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_POST_OPERATION 2024-11-24T22:15:57,478 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=118, state=SUCCESS, hasLock=false; CreateTableProcedure table=TestAcidGuarantees in 1.1470 sec 2024-11-24T22:15:58,480 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37701 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=118 2024-11-24T22:15:58,481 INFO [RPCClient-NioEventLoopGroup-4-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: CREATE, Table Name: default:TestAcidGuarantees completed 2024-11-24T22:15:58,482 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@55d1f058, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-24T22:15:58,482 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request bd6592c37bb2,37701,-1 for getting cluster id 2024-11-24T22:15:58,482 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-24T22:15:58,483 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '3c461ac1-06fb-4739-8702-6368e685b03c' 2024-11-24T22:15:58,483 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-24T22:15:58,483 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "3c461ac1-06fb-4739-8702-6368e685b03c" 2024-11-24T22:15:58,483 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@6d6a2c8a, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-24T22:15:58,483 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [bd6592c37bb2,37701,-1] 2024-11-24T22:15:58,484 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-24T22:15:58,484 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-24T22:15:58,485 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:57658, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-24T22:15:58,485 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@28595a6, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-24T22:15:58,485 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-24T22:15:58,486 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=bd6592c37bb2,40087,1732486423713, seqNum=-1] 2024-11-24T22:15:58,486 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-24T22:15:58,487 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:46782, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-24T22:15:58,487 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching master stub from registry 2024-11-24T22:15:58,488 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncConnectionImpl(321): The fetched master address is bd6592c37bb2,37701,1732486422992 2024-11-24T22:15:58,488 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.ConnectionUtils(555): The fetched master stub is org.apache.hadoop.hbase.shaded.protobuf.generated.MasterProtos$MasterService$Stub@615bccf7 2024-11-24T22:15:58,488 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=MasterService, sasl=false 2024-11-24T22:15:58,489 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:57674, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=MasterService 2024-11-24T22:15:58,490 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@581c4bd0, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-24T22:15:58,490 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request bd6592c37bb2,37701,-1 for getting cluster id 2024-11-24T22:15:58,490 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-24T22:15:58,491 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '3c461ac1-06fb-4739-8702-6368e685b03c' 2024-11-24T22:15:58,491 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-24T22:15:58,491 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "3c461ac1-06fb-4739-8702-6368e685b03c" 2024-11-24T22:15:58,491 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@1020b6b7, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-24T22:15:58,491 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [bd6592c37bb2,37701,-1] 2024-11-24T22:15:58,491 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-24T22:15:58,491 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-24T22:15:58,492 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:57698, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-24T22:15:58,492 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@79ff954, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-24T22:15:58,493 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@3692791b, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-24T22:15:58,493 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request bd6592c37bb2,37701,-1 for getting cluster id 2024-11-24T22:15:58,493 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-24T22:15:58,493 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '3c461ac1-06fb-4739-8702-6368e685b03c' 2024-11-24T22:15:58,493 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-24T22:15:58,493 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "3c461ac1-06fb-4739-8702-6368e685b03c" 2024-11-24T22:15:58,493 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@76663b98, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-24T22:15:58,493 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [bd6592c37bb2,37701,-1] 2024-11-24T22:15:58,494 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-24T22:15:58,494 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-24T22:15:58,494 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:57726, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-24T22:15:58,495 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@4824add2, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-24T22:15:58,495 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@2b791feb, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-24T22:15:58,495 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request bd6592c37bb2,37701,-1 for getting cluster id 2024-11-24T22:15:58,495 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-24T22:15:58,496 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '3c461ac1-06fb-4739-8702-6368e685b03c' 2024-11-24T22:15:58,496 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-24T22:15:58,496 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "3c461ac1-06fb-4739-8702-6368e685b03c" 2024-11-24T22:15:58,496 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@38eec39f, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-24T22:15:58,496 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [bd6592c37bb2,37701,-1] 2024-11-24T22:15:58,496 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-24T22:15:58,496 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-24T22:15:58,497 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:57744, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-24T22:15:58,497 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@1f0cb362, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-24T22:15:58,498 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@6da2e0be, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-24T22:15:58,498 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request bd6592c37bb2,37701,-1 for getting cluster id 2024-11-24T22:15:58,498 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-24T22:15:58,498 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '3c461ac1-06fb-4739-8702-6368e685b03c' 2024-11-24T22:15:58,498 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-24T22:15:58,498 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "3c461ac1-06fb-4739-8702-6368e685b03c" 2024-11-24T22:15:58,498 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@5d81725e, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-24T22:15:58,498 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [bd6592c37bb2,37701,-1] 2024-11-24T22:15:58,499 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-24T22:15:58,499 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-24T22:15:58,499 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:57754, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-24T22:15:58,500 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@47c46f10, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-24T22:15:58,500 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@3fc361f0, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-24T22:15:58,500 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request bd6592c37bb2,37701,-1 for getting cluster id 2024-11-24T22:15:58,500 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-24T22:15:58,501 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '3c461ac1-06fb-4739-8702-6368e685b03c' 2024-11-24T22:15:58,501 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-24T22:15:58,501 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "3c461ac1-06fb-4739-8702-6368e685b03c" 2024-11-24T22:15:58,501 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@7abe0988, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-24T22:15:58,501 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [bd6592c37bb2,37701,-1] 2024-11-24T22:15:58,501 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-24T22:15:58,501 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-24T22:15:58,502 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:57762, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-24T22:15:58,502 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@75acd72d, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-24T22:15:58,502 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@2f59c6a, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-24T22:15:58,502 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request bd6592c37bb2,37701,-1 for getting cluster id 2024-11-24T22:15:58,502 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-24T22:15:58,503 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '3c461ac1-06fb-4739-8702-6368e685b03c' 2024-11-24T22:15:58,503 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-24T22:15:58,503 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "3c461ac1-06fb-4739-8702-6368e685b03c" 2024-11-24T22:15:58,503 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@6ddbd9e9, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-24T22:15:58,503 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [bd6592c37bb2,37701,-1] 2024-11-24T22:15:58,503 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-24T22:15:58,504 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-24T22:15:58,504 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:57772, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-24T22:15:58,504 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@1ef1ce00, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-24T22:15:58,505 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@5ae789b8, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-24T22:15:58,505 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request bd6592c37bb2,37701,-1 for getting cluster id 2024-11-24T22:15:58,505 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-24T22:15:58,505 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '3c461ac1-06fb-4739-8702-6368e685b03c' 2024-11-24T22:15:58,506 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-24T22:15:58,506 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "3c461ac1-06fb-4739-8702-6368e685b03c" 2024-11-24T22:15:58,506 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@6950a328, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-24T22:15:58,506 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [bd6592c37bb2,37701,-1] 2024-11-24T22:15:58,506 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-24T22:15:58,506 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-24T22:15:58,507 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:57798, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-24T22:15:58,507 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@2c00ab33, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-24T22:15:58,507 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@5b7b3b89, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-24T22:15:58,507 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request bd6592c37bb2,37701,-1 for getting cluster id 2024-11-24T22:15:58,508 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-24T22:15:58,508 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '3c461ac1-06fb-4739-8702-6368e685b03c' 2024-11-24T22:15:58,508 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-24T22:15:58,508 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "3c461ac1-06fb-4739-8702-6368e685b03c" 2024-11-24T22:15:58,508 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@72619363, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-24T22:15:58,508 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [bd6592c37bb2,37701,-1] 2024-11-24T22:15:58,508 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-24T22:15:58,509 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-24T22:15:58,509 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:57820, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-24T22:15:58,509 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@3b906a8f, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-24T22:15:58,510 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@7e04f49c, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-24T22:15:58,510 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request bd6592c37bb2,37701,-1 for getting cluster id 2024-11-24T22:15:58,510 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-24T22:15:58,511 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '3c461ac1-06fb-4739-8702-6368e685b03c' 2024-11-24T22:15:58,511 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-24T22:15:58,511 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "3c461ac1-06fb-4739-8702-6368e685b03c" 2024-11-24T22:15:58,511 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@445a8d62, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-24T22:15:58,511 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [bd6592c37bb2,37701,-1] 2024-11-24T22:15:58,511 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-24T22:15:58,511 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-24T22:15:58,512 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:57838, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-24T22:15:58,512 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@34d09311, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-24T22:15:58,513 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@32ca199c, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-24T22:15:58,513 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request bd6592c37bb2,37701,-1 for getting cluster id 2024-11-24T22:15:58,513 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-24T22:15:58,513 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '3c461ac1-06fb-4739-8702-6368e685b03c' 2024-11-24T22:15:58,513 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-24T22:15:58,513 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "3c461ac1-06fb-4739-8702-6368e685b03c" 2024-11-24T22:15:58,513 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@394a20be, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-24T22:15:58,513 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [bd6592c37bb2,37701,-1] 2024-11-24T22:15:58,513 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-24T22:15:58,514 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-24T22:15:58,514 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:57860, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-24T22:15:58,514 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@5ef48068, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-24T22:15:58,515 DEBUG [Thread-2041 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-24T22:15:58,515 DEBUG [Thread-2032 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-24T22:15:58,515 DEBUG [Thread-2045 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-24T22:15:58,515 DEBUG [Thread-2049 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-24T22:15:58,515 DEBUG [Thread-2030 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-24T22:15:58,516 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37701 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-24T22:15:58,516 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=bd6592c37bb2,40087,1732486423713, seqNum=-1] 2024-11-24T22:15:58,516 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-24T22:15:58,516 DEBUG [Thread-2038 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-24T22:15:58,516 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=bd6592c37bb2,40087,1732486423713, seqNum=-1] 2024-11-24T22:15:58,516 DEBUG [Thread-2036 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-24T22:15:58,516 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=bd6592c37bb2,40087,1732486423713, seqNum=-1] 2024-11-24T22:15:58,516 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=bd6592c37bb2,40087,1732486423713, seqNum=-1] 2024-11-24T22:15:58,516 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-24T22:15:58,517 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-24T22:15:58,517 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-24T22:15:58,517 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=bd6592c37bb2,40087,1732486423713, seqNum=-1] 2024-11-24T22:15:58,517 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-24T22:15:58,517 DEBUG [Thread-2034 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-24T22:15:58,517 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:46784, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-24T22:15:58,517 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:46792, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-24T22:15:58,517 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:46800, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-24T22:15:58,517 DEBUG [Thread-2043 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-24T22:15:58,518 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:46798, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-24T22:15:58,518 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=bd6592c37bb2,40087,1732486423713, seqNum=-1] 2024-11-24T22:15:58,518 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37701 {}] procedure2.ProcedureExecutor(1139): Stored pid=121, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=121, table=TestAcidGuarantees 2024-11-24T22:15:58,518 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=bd6592c37bb2,40087,1732486423713, seqNum=-1] 2024-11-24T22:15:58,518 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:46816, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-24T22:15:58,518 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-24T22:15:58,518 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-24T22:15:58,519 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37701 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=121 2024-11-24T22:15:58,519 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:46824, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-24T22:15:58,519 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:46820, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-24T22:15:58,519 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=121, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=121, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-24T22:15:58,520 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='', locateType=CURRENT is [region=TestAcidGuarantees,,1732486556329.744bbec6934be304e01b009be760323d., hostname=bd6592c37bb2,40087,1732486423713, seqNum=2] 2024-11-24T22:15:58,520 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=121, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=121, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-24T22:15:58,520 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=122, ppid=121, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-24T22:15:58,520 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_2', locateType=CURRENT is [region=TestAcidGuarantees,,1732486556329.744bbec6934be304e01b009be760323d., hostname=bd6592c37bb2,40087,1732486423713, seqNum=2] 2024-11-24T22:15:58,520 DEBUG [Thread-2047 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-24T22:15:58,521 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=bd6592c37bb2,40087,1732486423713, seqNum=-1] 2024-11-24T22:15:58,521 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='', locateType=CURRENT is [region=TestAcidGuarantees,,1732486556329.744bbec6934be304e01b009be760323d., hostname=bd6592c37bb2,40087,1732486423713, seqNum=2] 2024-11-24T22:15:58,521 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-24T22:15:58,522 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='', locateType=CURRENT is [region=TestAcidGuarantees,,1732486556329.744bbec6934be304e01b009be760323d., hostname=bd6592c37bb2,40087,1732486423713, seqNum=2] 2024-11-24T22:15:58,522 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_2', locateType=CURRENT is [region=TestAcidGuarantees,,1732486556329.744bbec6934be304e01b009be760323d., hostname=bd6592c37bb2,40087,1732486423713, seqNum=2] 2024-11-24T22:15:58,523 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_0', locateType=CURRENT is [region=TestAcidGuarantees,,1732486556329.744bbec6934be304e01b009be760323d., hostname=bd6592c37bb2,40087,1732486423713, seqNum=2] 2024-11-24T22:15:58,523 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=bd6592c37bb2,40087,1732486423713, seqNum=-1] 2024-11-24T22:15:58,523 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=bd6592c37bb2,40087,1732486423713, seqNum=-1] 2024-11-24T22:15:58,523 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-24T22:15:58,523 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-24T22:15:58,524 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:46828, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-24T22:15:58,524 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_2', locateType=CURRENT is [region=TestAcidGuarantees,,1732486556329.744bbec6934be304e01b009be760323d., hostname=bd6592c37bb2,40087,1732486423713, seqNum=2] 2024-11-24T22:15:58,524 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:46842, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-24T22:15:58,525 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:46850, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-24T22:15:58,526 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_2', locateType=CURRENT is [region=TestAcidGuarantees,,1732486556329.744bbec6934be304e01b009be760323d., hostname=bd6592c37bb2,40087,1732486423713, seqNum=2] 2024-11-24T22:15:58,526 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='', locateType=CURRENT is [region=TestAcidGuarantees,,1732486556329.744bbec6934be304e01b009be760323d., hostname=bd6592c37bb2,40087,1732486423713, seqNum=2] 2024-11-24T22:15:58,527 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='', locateType=CURRENT is [region=TestAcidGuarantees,,1732486556329.744bbec6934be304e01b009be760323d., hostname=bd6592c37bb2,40087,1732486423713, seqNum=2] 2024-11-24T22:15:58,531 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 744bbec6934be304e01b009be760323d 3/3 column families, dataSize=46.96 KB heapSize=140.20 KB 2024-11-24T22:15:58,531 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40087 {}] regionserver.HRegion(8855): Flush requested on 744bbec6934be304e01b009be760323d 2024-11-24T22:15:58,533 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 744bbec6934be304e01b009be760323d, store=A 2024-11-24T22:15:58,533 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-24T22:15:58,533 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 744bbec6934be304e01b009be760323d, store=B 2024-11-24T22:15:58,533 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-24T22:15:58,533 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 744bbec6934be304e01b009be760323d, store=C 2024-11-24T22:15:58,533 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-24T22:15:58,557 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/744bbec6934be304e01b009be760323d/.tmp/A/56e7f0a11f784f479c172eeabafc39b7 is 50, key is test_row_0/A:col10/1732486558524/Put/seqid=0 2024-11-24T22:15:58,563 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45929 is added to blk_1073742278_1454 (size=14341) 2024-11-24T22:15:58,572 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40087 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=744bbec6934be304e01b009be760323d, server=bd6592c37bb2,40087,1732486423713 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-24T22:15:58,572 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40087 {}] ipc.CallRunner(138): callId: 7 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46792 deadline: 1732486618563, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=744bbec6934be304e01b009be760323d, server=bd6592c37bb2,40087,1732486423713 2024-11-24T22:15:58,573 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732486556329.744bbec6934be304e01b009be760323d., hostname=bd6592c37bb2,40087,1732486423713, seqNum=2 , the old value is region=TestAcidGuarantees,,1732486556329.744bbec6934be304e01b009be760323d., hostname=bd6592c37bb2,40087,1732486423713, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=744bbec6934be304e01b009be760323d, server=bd6592c37bb2,40087,1732486423713 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T22:15:58,573 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732486556329.744bbec6934be304e01b009be760323d., hostname=bd6592c37bb2,40087,1732486423713, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=744bbec6934be304e01b009be760323d, server=bd6592c37bb2,40087,1732486423713 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T22:15:58,573 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732486556329.744bbec6934be304e01b009be760323d., hostname=bd6592c37bb2,40087,1732486423713, seqNum=2 because the exception is null or not the one we care about 2024-11-24T22:15:58,573 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/bd6592c37bb2:0-3 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=209, cellSize=9565, totalHeapSize=21976, min timestamp=1732486558557, max timestamp=1732486558571Num uniques 859; 2024-11-24T22:15:58,574 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/bd6592c37bb2:0-3 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=9565, suffix heap size=117064,new segment heap size=21976 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=209 2024-11-24T22:15:58,574 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/bd6592c37bb2:0-4 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=209, cellSize=9565, totalHeapSize=21976, min timestamp=1732486558557, max timestamp=1732486558571Num uniques 859; 2024-11-24T22:15:58,574 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/bd6592c37bb2:0-5 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=209, cellSize=9565, totalHeapSize=21976, min timestamp=1732486558557, max timestamp=1732486558571Num uniques 859; 2024-11-24T22:15:58,574 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/bd6592c37bb2:0-5 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=9565, suffix heap size=117064,new segment heap size=21976 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=209 2024-11-24T22:15:58,574 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/bd6592c37bb2:0-4 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=9565, suffix heap size=117064,new segment heap size=21976 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=209 2024-11-24T22:15:58,620 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37701 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=121 2024-11-24T22:15:58,658 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40087 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=744bbec6934be304e01b009be760323d, server=bd6592c37bb2,40087,1732486423713 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-24T22:15:58,658 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40087 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46816 deadline: 1732486618645, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=744bbec6934be304e01b009be760323d, server=bd6592c37bb2,40087,1732486423713 2024-11-24T22:15:58,659 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40087 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=744bbec6934be304e01b009be760323d, server=bd6592c37bb2,40087,1732486423713 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-24T22:15:58,659 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732486556329.744bbec6934be304e01b009be760323d., hostname=bd6592c37bb2,40087,1732486423713, seqNum=2 , the old value is region=TestAcidGuarantees,,1732486556329.744bbec6934be304e01b009be760323d., hostname=bd6592c37bb2,40087,1732486423713, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=744bbec6934be304e01b009be760323d, server=bd6592c37bb2,40087,1732486423713 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T22:15:58,659 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40087 {}] ipc.CallRunner(138): callId: 8 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46842 deadline: 1732486618645, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=744bbec6934be304e01b009be760323d, server=bd6592c37bb2,40087,1732486423713 2024-11-24T22:15:58,659 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732486556329.744bbec6934be304e01b009be760323d., hostname=bd6592c37bb2,40087,1732486423713, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=744bbec6934be304e01b009be760323d, server=bd6592c37bb2,40087,1732486423713 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T22:15:58,659 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732486556329.744bbec6934be304e01b009be760323d., hostname=bd6592c37bb2,40087,1732486423713, seqNum=2 because the exception is null or not the one we care about 2024-11-24T22:15:58,659 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732486556329.744bbec6934be304e01b009be760323d., hostname=bd6592c37bb2,40087,1732486423713, seqNum=2 , the old value is region=TestAcidGuarantees,,1732486556329.744bbec6934be304e01b009be760323d., hostname=bd6592c37bb2,40087,1732486423713, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=744bbec6934be304e01b009be760323d, server=bd6592c37bb2,40087,1732486423713 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T22:15:58,659 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732486556329.744bbec6934be304e01b009be760323d., hostname=bd6592c37bb2,40087,1732486423713, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=744bbec6934be304e01b009be760323d, server=bd6592c37bb2,40087,1732486423713 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T22:15:58,659 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732486556329.744bbec6934be304e01b009be760323d., hostname=bd6592c37bb2,40087,1732486423713, seqNum=2 because the exception is null or not the one we care about 2024-11-24T22:15:58,659 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40087 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=744bbec6934be304e01b009be760323d, server=bd6592c37bb2,40087,1732486423713 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-24T22:15:58,660 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40087 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46824 deadline: 1732486618646, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=744bbec6934be304e01b009be760323d, server=bd6592c37bb2,40087,1732486423713 2024-11-24T22:15:58,660 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732486556329.744bbec6934be304e01b009be760323d., hostname=bd6592c37bb2,40087,1732486423713, seqNum=2 , the old value is region=TestAcidGuarantees,,1732486556329.744bbec6934be304e01b009be760323d., hostname=bd6592c37bb2,40087,1732486423713, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=744bbec6934be304e01b009be760323d, server=bd6592c37bb2,40087,1732486423713 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T22:15:58,660 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40087 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=744bbec6934be304e01b009be760323d, server=bd6592c37bb2,40087,1732486423713 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-24T22:15:58,660 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732486556329.744bbec6934be304e01b009be760323d., hostname=bd6592c37bb2,40087,1732486423713, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=744bbec6934be304e01b009be760323d, server=bd6592c37bb2,40087,1732486423713 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T22:15:58,660 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732486556329.744bbec6934be304e01b009be760323d., hostname=bd6592c37bb2,40087,1732486423713, seqNum=2 because the exception is null or not the one we care about 2024-11-24T22:15:58,660 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40087 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46820 deadline: 1732486618649, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=744bbec6934be304e01b009be760323d, server=bd6592c37bb2,40087,1732486423713 2024-11-24T22:15:58,660 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732486556329.744bbec6934be304e01b009be760323d., hostname=bd6592c37bb2,40087,1732486423713, seqNum=2 , the old value is region=TestAcidGuarantees,,1732486556329.744bbec6934be304e01b009be760323d., hostname=bd6592c37bb2,40087,1732486423713, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=744bbec6934be304e01b009be760323d, server=bd6592c37bb2,40087,1732486423713 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T22:15:58,660 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732486556329.744bbec6934be304e01b009be760323d., hostname=bd6592c37bb2,40087,1732486423713, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=744bbec6934be304e01b009be760323d, server=bd6592c37bb2,40087,1732486423713 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T22:15:58,660 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732486556329.744bbec6934be304e01b009be760323d., hostname=bd6592c37bb2,40087,1732486423713, seqNum=2 because the exception is null or not the one we care about 2024-11-24T22:15:58,673 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=40087 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=122 2024-11-24T22:15:58,674 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bd6592c37bb2:0-0 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732486556329.744bbec6934be304e01b009be760323d. 2024-11-24T22:15:58,674 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bd6592c37bb2:0-0 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732486556329.744bbec6934be304e01b009be760323d. as already flushing 2024-11-24T22:15:58,674 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bd6592c37bb2:0-0 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732486556329.744bbec6934be304e01b009be760323d. 2024-11-24T22:15:58,674 ERROR [RS_FLUSH_OPERATIONS-regionserver/bd6592c37bb2:0-0 {event_type=RS_FLUSH_REGIONS, pid=122}] handler.RSProcedureHandler(58): pid=122 java.io.IOException: Unable to complete flush {ENCODED => 744bbec6934be304e01b009be760323d, NAME => 'TestAcidGuarantees,,1732486556329.744bbec6934be304e01b009be760323d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T22:15:58,674 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bd6592c37bb2:0-0 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=122 java.io.IOException: Unable to complete flush {ENCODED => 744bbec6934be304e01b009be760323d, NAME => 'TestAcidGuarantees,,1732486556329.744bbec6934be304e01b009be760323d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T22:15:58,675 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37701 {}] master.HMaster(4177): Remote procedure failed, pid=122 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 744bbec6934be304e01b009be760323d, NAME => 'TestAcidGuarantees,,1732486556329.744bbec6934be304e01b009be760323d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 744bbec6934be304e01b009be760323d, NAME => 'TestAcidGuarantees,,1732486556329.744bbec6934be304e01b009be760323d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T22:15:58,690 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40087 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=744bbec6934be304e01b009be760323d, server=bd6592c37bb2,40087,1732486423713 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-24T22:15:58,690 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40087 {}] ipc.CallRunner(138): callId: 8 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46792 deadline: 1732486618681, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=744bbec6934be304e01b009be760323d, server=bd6592c37bb2,40087,1732486423713 2024-11-24T22:15:58,690 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732486556329.744bbec6934be304e01b009be760323d., hostname=bd6592c37bb2,40087,1732486423713, seqNum=2 , the old value is region=TestAcidGuarantees,,1732486556329.744bbec6934be304e01b009be760323d., hostname=bd6592c37bb2,40087,1732486423713, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=744bbec6934be304e01b009be760323d, server=bd6592c37bb2,40087,1732486423713 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T22:15:58,690 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732486556329.744bbec6934be304e01b009be760323d., hostname=bd6592c37bb2,40087,1732486423713, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=744bbec6934be304e01b009be760323d, server=bd6592c37bb2,40087,1732486423713 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T22:15:58,690 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732486556329.744bbec6934be304e01b009be760323d., hostname=bd6592c37bb2,40087,1732486423713, seqNum=2 because the exception is null or not the one we care about 2024-11-24T22:15:58,781 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40087 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=744bbec6934be304e01b009be760323d, server=bd6592c37bb2,40087,1732486423713 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-24T22:15:58,781 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40087 {}] ipc.CallRunner(138): callId: 11 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46816 deadline: 1732486618771, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=744bbec6934be304e01b009be760323d, server=bd6592c37bb2,40087,1732486423713 2024-11-24T22:15:58,781 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732486556329.744bbec6934be304e01b009be760323d., hostname=bd6592c37bb2,40087,1732486423713, seqNum=2 , the old value is region=TestAcidGuarantees,,1732486556329.744bbec6934be304e01b009be760323d., hostname=bd6592c37bb2,40087,1732486423713, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=744bbec6934be304e01b009be760323d, server=bd6592c37bb2,40087,1732486423713 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T22:15:58,781 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732486556329.744bbec6934be304e01b009be760323d., hostname=bd6592c37bb2,40087,1732486423713, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=744bbec6934be304e01b009be760323d, server=bd6592c37bb2,40087,1732486423713 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T22:15:58,781 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40087 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=744bbec6934be304e01b009be760323d, server=bd6592c37bb2,40087,1732486423713 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-24T22:15:58,781 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732486556329.744bbec6934be304e01b009be760323d., hostname=bd6592c37bb2,40087,1732486423713, seqNum=2 because the exception is null or not the one we care about 2024-11-24T22:15:58,782 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40087 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46842 deadline: 1732486618771, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=744bbec6934be304e01b009be760323d, server=bd6592c37bb2,40087,1732486423713 2024-11-24T22:15:58,782 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40087 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=744bbec6934be304e01b009be760323d, server=bd6592c37bb2,40087,1732486423713 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-24T22:15:58,782 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732486556329.744bbec6934be304e01b009be760323d., hostname=bd6592c37bb2,40087,1732486423713, seqNum=2 , the old value is region=TestAcidGuarantees,,1732486556329.744bbec6934be304e01b009be760323d., hostname=bd6592c37bb2,40087,1732486423713, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=744bbec6934be304e01b009be760323d, server=bd6592c37bb2,40087,1732486423713 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T22:15:58,782 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732486556329.744bbec6934be304e01b009be760323d., hostname=bd6592c37bb2,40087,1732486423713, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=744bbec6934be304e01b009be760323d, server=bd6592c37bb2,40087,1732486423713 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T22:15:58,782 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40087 {}] ipc.CallRunner(138): callId: 11 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46824 deadline: 1732486618771, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=744bbec6934be304e01b009be760323d, server=bd6592c37bb2,40087,1732486423713 2024-11-24T22:15:58,782 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732486556329.744bbec6934be304e01b009be760323d., hostname=bd6592c37bb2,40087,1732486423713, seqNum=2 because the exception is null or not the one we care about 2024-11-24T22:15:58,782 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40087 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=744bbec6934be304e01b009be760323d, server=bd6592c37bb2,40087,1732486423713 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-24T22:15:58,782 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732486556329.744bbec6934be304e01b009be760323d., hostname=bd6592c37bb2,40087,1732486423713, seqNum=2 , the old value is region=TestAcidGuarantees,,1732486556329.744bbec6934be304e01b009be760323d., hostname=bd6592c37bb2,40087,1732486423713, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=744bbec6934be304e01b009be760323d, server=bd6592c37bb2,40087,1732486423713 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T22:15:58,782 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732486556329.744bbec6934be304e01b009be760323d., hostname=bd6592c37bb2,40087,1732486423713, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=744bbec6934be304e01b009be760323d, server=bd6592c37bb2,40087,1732486423713 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T22:15:58,782 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40087 {}] ipc.CallRunner(138): callId: 11 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46820 deadline: 1732486618771, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=744bbec6934be304e01b009be760323d, server=bd6592c37bb2,40087,1732486423713 2024-11-24T22:15:58,782 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732486556329.744bbec6934be304e01b009be760323d., hostname=bd6592c37bb2,40087,1732486423713, seqNum=2 because the exception is null or not the one we care about 2024-11-24T22:15:58,782 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732486556329.744bbec6934be304e01b009be760323d., hostname=bd6592c37bb2,40087,1732486423713, seqNum=2 , the old value is region=TestAcidGuarantees,,1732486556329.744bbec6934be304e01b009be760323d., hostname=bd6592c37bb2,40087,1732486423713, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=744bbec6934be304e01b009be760323d, server=bd6592c37bb2,40087,1732486423713 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T22:15:58,783 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732486556329.744bbec6934be304e01b009be760323d., hostname=bd6592c37bb2,40087,1732486423713, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=744bbec6934be304e01b009be760323d, server=bd6592c37bb2,40087,1732486423713 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T22:15:58,783 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732486556329.744bbec6934be304e01b009be760323d., hostname=bd6592c37bb2,40087,1732486423713, seqNum=2 because the exception is null or not the one we care about 2024-11-24T22:15:58,826 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=40087 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=122 2024-11-24T22:15:58,827 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bd6592c37bb2:0-1 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732486556329.744bbec6934be304e01b009be760323d. 2024-11-24T22:15:58,827 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bd6592c37bb2:0-1 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732486556329.744bbec6934be304e01b009be760323d. as already flushing 2024-11-24T22:15:58,827 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bd6592c37bb2:0-1 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732486556329.744bbec6934be304e01b009be760323d. 2024-11-24T22:15:58,827 ERROR [RS_FLUSH_OPERATIONS-regionserver/bd6592c37bb2:0-1 {event_type=RS_FLUSH_REGIONS, pid=122}] handler.RSProcedureHandler(58): pid=122 java.io.IOException: Unable to complete flush {ENCODED => 744bbec6934be304e01b009be760323d, NAME => 'TestAcidGuarantees,,1732486556329.744bbec6934be304e01b009be760323d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T22:15:58,827 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bd6592c37bb2:0-1 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=122 java.io.IOException: Unable to complete flush {ENCODED => 744bbec6934be304e01b009be760323d, NAME => 'TestAcidGuarantees,,1732486556329.744bbec6934be304e01b009be760323d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T22:15:58,828 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37701 {}] master.HMaster(4177): Remote procedure failed, pid=122 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 744bbec6934be304e01b009be760323d, NAME => 'TestAcidGuarantees,,1732486556329.744bbec6934be304e01b009be760323d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 744bbec6934be304e01b009be760323d, NAME => 'TestAcidGuarantees,,1732486556329.744bbec6934be304e01b009be760323d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T22:15:58,831 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37701 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=121 2024-11-24T22:15:58,910 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40087 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=744bbec6934be304e01b009be760323d, server=bd6592c37bb2,40087,1732486423713 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-24T22:15:58,910 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40087 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46792 deadline: 1732486618900, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=744bbec6934be304e01b009be760323d, server=bd6592c37bb2,40087,1732486423713 2024-11-24T22:15:58,911 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732486556329.744bbec6934be304e01b009be760323d., hostname=bd6592c37bb2,40087,1732486423713, seqNum=2 , the old value is region=TestAcidGuarantees,,1732486556329.744bbec6934be304e01b009be760323d., hostname=bd6592c37bb2,40087,1732486423713, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=744bbec6934be304e01b009be760323d, server=bd6592c37bb2,40087,1732486423713 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T22:15:58,911 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732486556329.744bbec6934be304e01b009be760323d., hostname=bd6592c37bb2,40087,1732486423713, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=744bbec6934be304e01b009be760323d, server=bd6592c37bb2,40087,1732486423713 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T22:15:58,911 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732486556329.744bbec6934be304e01b009be760323d., hostname=bd6592c37bb2,40087,1732486423713, seqNum=2 because the exception is null or not the one we care about 2024-11-24T22:15:58,963 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=12 (bloomFilter=true), to=hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/744bbec6934be304e01b009be760323d/.tmp/A/56e7f0a11f784f479c172eeabafc39b7 2024-11-24T22:15:58,980 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=40087 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=122 2024-11-24T22:15:58,980 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bd6592c37bb2:0-2 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732486556329.744bbec6934be304e01b009be760323d. 2024-11-24T22:15:58,980 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bd6592c37bb2:0-2 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732486556329.744bbec6934be304e01b009be760323d. as already flushing 2024-11-24T22:15:58,980 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bd6592c37bb2:0-2 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732486556329.744bbec6934be304e01b009be760323d. 2024-11-24T22:15:58,980 ERROR [RS_FLUSH_OPERATIONS-regionserver/bd6592c37bb2:0-2 {event_type=RS_FLUSH_REGIONS, pid=122}] handler.RSProcedureHandler(58): pid=122 java.io.IOException: Unable to complete flush {ENCODED => 744bbec6934be304e01b009be760323d, NAME => 'TestAcidGuarantees,,1732486556329.744bbec6934be304e01b009be760323d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T22:15:58,980 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bd6592c37bb2:0-2 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=122 java.io.IOException: Unable to complete flush {ENCODED => 744bbec6934be304e01b009be760323d, NAME => 'TestAcidGuarantees,,1732486556329.744bbec6934be304e01b009be760323d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T22:15:58,981 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37701 {}] master.HMaster(4177): Remote procedure failed, pid=122 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 744bbec6934be304e01b009be760323d, NAME => 'TestAcidGuarantees,,1732486556329.744bbec6934be304e01b009be760323d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 744bbec6934be304e01b009be760323d, NAME => 'TestAcidGuarantees,,1732486556329.744bbec6934be304e01b009be760323d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T22:15:58,987 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/744bbec6934be304e01b009be760323d/.tmp/B/6f1957bed3164d5aa2811fb5eb436e11 is 50, key is test_row_0/B:col10/1732486558524/Put/seqid=0 2024-11-24T22:15:58,993 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45929 is added to blk_1073742279_1455 (size=12001) 2024-11-24T22:15:59,001 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40087 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=744bbec6934be304e01b009be760323d, server=bd6592c37bb2,40087,1732486423713 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-24T22:15:59,001 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40087 {}] ipc.CallRunner(138): callId: 12 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46816 deadline: 1732486618991, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=744bbec6934be304e01b009be760323d, server=bd6592c37bb2,40087,1732486423713 2024-11-24T22:15:59,001 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732486556329.744bbec6934be304e01b009be760323d., hostname=bd6592c37bb2,40087,1732486423713, seqNum=2 , the old value is region=TestAcidGuarantees,,1732486556329.744bbec6934be304e01b009be760323d., hostname=bd6592c37bb2,40087,1732486423713, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=744bbec6934be304e01b009be760323d, server=bd6592c37bb2,40087,1732486423713 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T22:15:59,001 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732486556329.744bbec6934be304e01b009be760323d., hostname=bd6592c37bb2,40087,1732486423713, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=744bbec6934be304e01b009be760323d, server=bd6592c37bb2,40087,1732486423713 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T22:15:59,001 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40087 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=744bbec6934be304e01b009be760323d, server=bd6592c37bb2,40087,1732486423713 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-24T22:15:59,001 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732486556329.744bbec6934be304e01b009be760323d., hostname=bd6592c37bb2,40087,1732486423713, seqNum=2 because the exception is null or not the one we care about 2024-11-24T22:15:59,002 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40087 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46842 deadline: 1732486618991, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=744bbec6934be304e01b009be760323d, server=bd6592c37bb2,40087,1732486423713 2024-11-24T22:15:59,002 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732486556329.744bbec6934be304e01b009be760323d., hostname=bd6592c37bb2,40087,1732486423713, seqNum=2 , the old value is region=TestAcidGuarantees,,1732486556329.744bbec6934be304e01b009be760323d., hostname=bd6592c37bb2,40087,1732486423713, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=744bbec6934be304e01b009be760323d, server=bd6592c37bb2,40087,1732486423713 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T22:15:59,002 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40087 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=744bbec6934be304e01b009be760323d, server=bd6592c37bb2,40087,1732486423713 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-24T22:15:59,002 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732486556329.744bbec6934be304e01b009be760323d., hostname=bd6592c37bb2,40087,1732486423713, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=744bbec6934be304e01b009be760323d, server=bd6592c37bb2,40087,1732486423713 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T22:15:59,002 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732486556329.744bbec6934be304e01b009be760323d., hostname=bd6592c37bb2,40087,1732486423713, seqNum=2 because the exception is null or not the one we care about 2024-11-24T22:15:59,002 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40087 {}] ipc.CallRunner(138): callId: 12 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46824 deadline: 1732486618991, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=744bbec6934be304e01b009be760323d, server=bd6592c37bb2,40087,1732486423713 2024-11-24T22:15:59,002 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732486556329.744bbec6934be304e01b009be760323d., hostname=bd6592c37bb2,40087,1732486423713, seqNum=2 , the old value is region=TestAcidGuarantees,,1732486556329.744bbec6934be304e01b009be760323d., hostname=bd6592c37bb2,40087,1732486423713, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=744bbec6934be304e01b009be760323d, server=bd6592c37bb2,40087,1732486423713 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T22:15:59,002 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732486556329.744bbec6934be304e01b009be760323d., hostname=bd6592c37bb2,40087,1732486423713, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=744bbec6934be304e01b009be760323d, server=bd6592c37bb2,40087,1732486423713 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T22:15:59,002 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40087 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=744bbec6934be304e01b009be760323d, server=bd6592c37bb2,40087,1732486423713 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-24T22:15:59,002 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732486556329.744bbec6934be304e01b009be760323d., hostname=bd6592c37bb2,40087,1732486423713, seqNum=2 because the exception is null or not the one we care about 2024-11-24T22:15:59,002 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40087 {}] ipc.CallRunner(138): callId: 12 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46820 deadline: 1732486618992, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=744bbec6934be304e01b009be760323d, server=bd6592c37bb2,40087,1732486423713 2024-11-24T22:15:59,003 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732486556329.744bbec6934be304e01b009be760323d., hostname=bd6592c37bb2,40087,1732486423713, seqNum=2 , the old value is region=TestAcidGuarantees,,1732486556329.744bbec6934be304e01b009be760323d., hostname=bd6592c37bb2,40087,1732486423713, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=744bbec6934be304e01b009be760323d, server=bd6592c37bb2,40087,1732486423713 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T22:15:59,003 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732486556329.744bbec6934be304e01b009be760323d., hostname=bd6592c37bb2,40087,1732486423713, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=744bbec6934be304e01b009be760323d, server=bd6592c37bb2,40087,1732486423713 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T22:15:59,003 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732486556329.744bbec6934be304e01b009be760323d., hostname=bd6592c37bb2,40087,1732486423713, seqNum=2 because the exception is null or not the one we care about 2024-11-24T22:15:59,132 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=40087 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=122 2024-11-24T22:15:59,133 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bd6592c37bb2:0-0 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732486556329.744bbec6934be304e01b009be760323d. 2024-11-24T22:15:59,133 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bd6592c37bb2:0-0 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732486556329.744bbec6934be304e01b009be760323d. as already flushing 2024-11-24T22:15:59,133 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bd6592c37bb2:0-0 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732486556329.744bbec6934be304e01b009be760323d. 2024-11-24T22:15:59,133 ERROR [RS_FLUSH_OPERATIONS-regionserver/bd6592c37bb2:0-0 {event_type=RS_FLUSH_REGIONS, pid=122}] handler.RSProcedureHandler(58): pid=122 java.io.IOException: Unable to complete flush {ENCODED => 744bbec6934be304e01b009be760323d, NAME => 'TestAcidGuarantees,,1732486556329.744bbec6934be304e01b009be760323d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T22:15:59,133 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bd6592c37bb2:0-0 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=122 java.io.IOException: Unable to complete flush {ENCODED => 744bbec6934be304e01b009be760323d, NAME => 'TestAcidGuarantees,,1732486556329.744bbec6934be304e01b009be760323d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T22:15:59,133 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37701 {}] master.HMaster(4177): Remote procedure failed, pid=122 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 744bbec6934be304e01b009be760323d, NAME => 'TestAcidGuarantees,,1732486556329.744bbec6934be304e01b009be760323d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 744bbec6934be304e01b009be760323d, NAME => 'TestAcidGuarantees,,1732486556329.744bbec6934be304e01b009be760323d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T22:15:59,141 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37701 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=121 2024-11-24T22:15:59,228 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40087 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=744bbec6934be304e01b009be760323d, server=bd6592c37bb2,40087,1732486423713 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-24T22:15:59,228 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40087 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46792 deadline: 1732486619220, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=744bbec6934be304e01b009be760323d, server=bd6592c37bb2,40087,1732486423713 2024-11-24T22:15:59,229 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732486556329.744bbec6934be304e01b009be760323d., hostname=bd6592c37bb2,40087,1732486423713, seqNum=2 , the old value is region=TestAcidGuarantees,,1732486556329.744bbec6934be304e01b009be760323d., hostname=bd6592c37bb2,40087,1732486423713, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=744bbec6934be304e01b009be760323d, server=bd6592c37bb2,40087,1732486423713 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T22:15:59,229 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732486556329.744bbec6934be304e01b009be760323d., hostname=bd6592c37bb2,40087,1732486423713, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=744bbec6934be304e01b009be760323d, server=bd6592c37bb2,40087,1732486423713 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T22:15:59,229 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732486556329.744bbec6934be304e01b009be760323d., hostname=bd6592c37bb2,40087,1732486423713, seqNum=2 because the exception is null or not the one we care about 2024-11-24T22:15:59,285 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=40087 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=122 2024-11-24T22:15:59,285 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bd6592c37bb2:0-1 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732486556329.744bbec6934be304e01b009be760323d. 2024-11-24T22:15:59,285 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bd6592c37bb2:0-1 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732486556329.744bbec6934be304e01b009be760323d. as already flushing 2024-11-24T22:15:59,286 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bd6592c37bb2:0-1 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732486556329.744bbec6934be304e01b009be760323d. 2024-11-24T22:15:59,286 ERROR [RS_FLUSH_OPERATIONS-regionserver/bd6592c37bb2:0-1 {event_type=RS_FLUSH_REGIONS, pid=122}] handler.RSProcedureHandler(58): pid=122 java.io.IOException: Unable to complete flush {ENCODED => 744bbec6934be304e01b009be760323d, NAME => 'TestAcidGuarantees,,1732486556329.744bbec6934be304e01b009be760323d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T22:15:59,286 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bd6592c37bb2:0-1 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=122 java.io.IOException: Unable to complete flush {ENCODED => 744bbec6934be304e01b009be760323d, NAME => 'TestAcidGuarantees,,1732486556329.744bbec6934be304e01b009be760323d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T22:15:59,286 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37701 {}] master.HMaster(4177): Remote procedure failed, pid=122 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 744bbec6934be304e01b009be760323d, NAME => 'TestAcidGuarantees,,1732486556329.744bbec6934be304e01b009be760323d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 744bbec6934be304e01b009be760323d, NAME => 'TestAcidGuarantees,,1732486556329.744bbec6934be304e01b009be760323d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T22:15:59,316 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40087 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=744bbec6934be304e01b009be760323d, server=bd6592c37bb2,40087,1732486423713 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-24T22:15:59,316 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40087 {}] ipc.CallRunner(138): callId: 13 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46816 deadline: 1732486619311, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=744bbec6934be304e01b009be760323d, server=bd6592c37bb2,40087,1732486423713 2024-11-24T22:15:59,316 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732486556329.744bbec6934be304e01b009be760323d., hostname=bd6592c37bb2,40087,1732486423713, seqNum=2 , the old value is region=TestAcidGuarantees,,1732486556329.744bbec6934be304e01b009be760323d., hostname=bd6592c37bb2,40087,1732486423713, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=744bbec6934be304e01b009be760323d, server=bd6592c37bb2,40087,1732486423713 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T22:15:59,316 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40087 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=744bbec6934be304e01b009be760323d, server=bd6592c37bb2,40087,1732486423713 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-24T22:15:59,316 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732486556329.744bbec6934be304e01b009be760323d., hostname=bd6592c37bb2,40087,1732486423713, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=744bbec6934be304e01b009be760323d, server=bd6592c37bb2,40087,1732486423713 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T22:15:59,317 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732486556329.744bbec6934be304e01b009be760323d., hostname=bd6592c37bb2,40087,1732486423713, seqNum=2 because the exception is null or not the one we care about 2024-11-24T22:15:59,317 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40087 {}] ipc.CallRunner(138): callId: 11 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46842 deadline: 1732486619311, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=744bbec6934be304e01b009be760323d, server=bd6592c37bb2,40087,1732486423713 2024-11-24T22:15:59,317 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40087 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=744bbec6934be304e01b009be760323d, server=bd6592c37bb2,40087,1732486423713 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-24T22:15:59,317 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732486556329.744bbec6934be304e01b009be760323d., hostname=bd6592c37bb2,40087,1732486423713, seqNum=2 , the old value is region=TestAcidGuarantees,,1732486556329.744bbec6934be304e01b009be760323d., hostname=bd6592c37bb2,40087,1732486423713, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=744bbec6934be304e01b009be760323d, server=bd6592c37bb2,40087,1732486423713 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T22:15:59,317 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40087 {}] ipc.CallRunner(138): callId: 13 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46824 deadline: 1732486619311, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=744bbec6934be304e01b009be760323d, server=bd6592c37bb2,40087,1732486423713 2024-11-24T22:15:59,317 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732486556329.744bbec6934be304e01b009be760323d., hostname=bd6592c37bb2,40087,1732486423713, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=744bbec6934be304e01b009be760323d, server=bd6592c37bb2,40087,1732486423713 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T22:15:59,317 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732486556329.744bbec6934be304e01b009be760323d., hostname=bd6592c37bb2,40087,1732486423713, seqNum=2 because the exception is null or not the one we care about 2024-11-24T22:15:59,317 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732486556329.744bbec6934be304e01b009be760323d., hostname=bd6592c37bb2,40087,1732486423713, seqNum=2 , the old value is region=TestAcidGuarantees,,1732486556329.744bbec6934be304e01b009be760323d., hostname=bd6592c37bb2,40087,1732486423713, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=744bbec6934be304e01b009be760323d, server=bd6592c37bb2,40087,1732486423713 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T22:15:59,317 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732486556329.744bbec6934be304e01b009be760323d., hostname=bd6592c37bb2,40087,1732486423713, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=744bbec6934be304e01b009be760323d, server=bd6592c37bb2,40087,1732486423713 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T22:15:59,317 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732486556329.744bbec6934be304e01b009be760323d., hostname=bd6592c37bb2,40087,1732486423713, seqNum=2 because the exception is null or not the one we care about 2024-11-24T22:15:59,324 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40087 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=744bbec6934be304e01b009be760323d, server=bd6592c37bb2,40087,1732486423713 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-24T22:15:59,324 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40087 {}] ipc.CallRunner(138): callId: 13 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46820 deadline: 1732486619312, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=744bbec6934be304e01b009be760323d, server=bd6592c37bb2,40087,1732486423713 2024-11-24T22:15:59,325 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732486556329.744bbec6934be304e01b009be760323d., hostname=bd6592c37bb2,40087,1732486423713, seqNum=2 , the old value is region=TestAcidGuarantees,,1732486556329.744bbec6934be304e01b009be760323d., hostname=bd6592c37bb2,40087,1732486423713, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=744bbec6934be304e01b009be760323d, server=bd6592c37bb2,40087,1732486423713 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T22:15:59,325 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732486556329.744bbec6934be304e01b009be760323d., hostname=bd6592c37bb2,40087,1732486423713, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=744bbec6934be304e01b009be760323d, server=bd6592c37bb2,40087,1732486423713 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T22:15:59,325 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732486556329.744bbec6934be304e01b009be760323d., hostname=bd6592c37bb2,40087,1732486423713, seqNum=2 because the exception is null or not the one we care about 2024-11-24T22:15:59,394 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=12 (bloomFilter=true), to=hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/744bbec6934be304e01b009be760323d/.tmp/B/6f1957bed3164d5aa2811fb5eb436e11 2024-11-24T22:15:59,413 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/744bbec6934be304e01b009be760323d/.tmp/C/d9a1c99c3bfb401092039d1293ad1252 is 50, key is test_row_0/C:col10/1732486558524/Put/seqid=0 2024-11-24T22:15:59,417 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45929 is added to blk_1073742280_1456 (size=12001) 2024-11-24T22:15:59,418 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=12 (bloomFilter=true), to=hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/744bbec6934be304e01b009be760323d/.tmp/C/d9a1c99c3bfb401092039d1293ad1252 2024-11-24T22:15:59,421 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/744bbec6934be304e01b009be760323d/.tmp/A/56e7f0a11f784f479c172eeabafc39b7 as hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/744bbec6934be304e01b009be760323d/A/56e7f0a11f784f479c172eeabafc39b7 2024-11-24T22:15:59,425 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/744bbec6934be304e01b009be760323d/A/56e7f0a11f784f479c172eeabafc39b7, entries=200, sequenceid=12, filesize=14.0 K 2024-11-24T22:15:59,426 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/744bbec6934be304e01b009be760323d/.tmp/B/6f1957bed3164d5aa2811fb5eb436e11 as hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/744bbec6934be304e01b009be760323d/B/6f1957bed3164d5aa2811fb5eb436e11 2024-11-24T22:15:59,429 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/744bbec6934be304e01b009be760323d/B/6f1957bed3164d5aa2811fb5eb436e11, entries=150, sequenceid=12, filesize=11.7 K 2024-11-24T22:15:59,430 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/744bbec6934be304e01b009be760323d/.tmp/C/d9a1c99c3bfb401092039d1293ad1252 as hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/744bbec6934be304e01b009be760323d/C/d9a1c99c3bfb401092039d1293ad1252 2024-11-24T22:15:59,432 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/744bbec6934be304e01b009be760323d/C/d9a1c99c3bfb401092039d1293ad1252, entries=150, sequenceid=12, filesize=11.7 K 2024-11-24T22:15:59,433 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~53.67 KB/54960, heapSize ~160.08 KB/163920, currentSize=127.47 KB/130530 for 744bbec6934be304e01b009be760323d in 902ms, sequenceid=12, compaction requested=false 2024-11-24T22:15:59,433 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 744bbec6934be304e01b009be760323d: 2024-11-24T22:15:59,438 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=40087 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=122 2024-11-24T22:15:59,438 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bd6592c37bb2:0-2 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732486556329.744bbec6934be304e01b009be760323d. 2024-11-24T22:15:59,438 INFO [RS_FLUSH_OPERATIONS-regionserver/bd6592c37bb2:0-2 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.HRegion(2902): Flushing 744bbec6934be304e01b009be760323d 3/3 column families, dataSize=127.47 KB heapSize=360.38 KB 2024-11-24T22:15:59,439 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bd6592c37bb2:0-2 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 744bbec6934be304e01b009be760323d, store=A 2024-11-24T22:15:59,439 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bd6592c37bb2:0-2 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-24T22:15:59,439 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bd6592c37bb2:0-2 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 744bbec6934be304e01b009be760323d, store=B 2024-11-24T22:15:59,439 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bd6592c37bb2:0-2 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-24T22:15:59,439 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bd6592c37bb2:0-2 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 744bbec6934be304e01b009be760323d, store=C 2024-11-24T22:15:59,439 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bd6592c37bb2:0-2 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-24T22:15:59,443 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bd6592c37bb2:0-2 {event_type=RS_FLUSH_REGIONS, pid=122}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/744bbec6934be304e01b009be760323d/.tmp/A/9338a57d248a4d55bbcb179d0bffa465 is 50, key is test_row_0/A:col10/1732486558630/Put/seqid=0 2024-11-24T22:15:59,446 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45929 is added to blk_1073742281_1457 (size=12001) 2024-11-24T22:15:59,447 INFO [RS_FLUSH_OPERATIONS-regionserver/bd6592c37bb2:0-2 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=42.49 KB at sequenceid=47 (bloomFilter=true), to=hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/744bbec6934be304e01b009be760323d/.tmp/A/9338a57d248a4d55bbcb179d0bffa465 2024-11-24T22:15:59,453 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bd6592c37bb2:0-2 {event_type=RS_FLUSH_REGIONS, pid=122}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/744bbec6934be304e01b009be760323d/.tmp/B/6d14ee73785241eca44e62a589b062d7 is 50, key is test_row_0/B:col10/1732486558630/Put/seqid=0 2024-11-24T22:15:59,458 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45929 is added to blk_1073742282_1458 (size=12001) 2024-11-24T22:15:59,651 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37701 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=121 2024-11-24T22:15:59,750 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40087 {}] regionserver.HRegion(8855): Flush requested on 744bbec6934be304e01b009be760323d 2024-11-24T22:15:59,750 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732486556329.744bbec6934be304e01b009be760323d. as already flushing 2024-11-24T22:15:59,806 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40087 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=744bbec6934be304e01b009be760323d, server=bd6592c37bb2,40087,1732486423713 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-24T22:15:59,806 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40087 {}] ipc.CallRunner(138): callId: 19 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46792 deadline: 1732486619800, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=744bbec6934be304e01b009be760323d, server=bd6592c37bb2,40087,1732486423713 2024-11-24T22:15:59,806 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732486556329.744bbec6934be304e01b009be760323d., hostname=bd6592c37bb2,40087,1732486423713, seqNum=2 , the old value is region=TestAcidGuarantees,,1732486556329.744bbec6934be304e01b009be760323d., hostname=bd6592c37bb2,40087,1732486423713, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=744bbec6934be304e01b009be760323d, server=bd6592c37bb2,40087,1732486423713 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T22:15:59,806 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732486556329.744bbec6934be304e01b009be760323d., hostname=bd6592c37bb2,40087,1732486423713, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=744bbec6934be304e01b009be760323d, server=bd6592c37bb2,40087,1732486423713 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T22:15:59,806 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732486556329.744bbec6934be304e01b009be760323d., hostname=bd6592c37bb2,40087,1732486423713, seqNum=2 because the exception is null or not the one we care about 2024-11-24T22:15:59,830 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40087 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=744bbec6934be304e01b009be760323d, server=bd6592c37bb2,40087,1732486423713 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-24T22:15:59,830 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40087 {}] ipc.CallRunner(138): callId: 12 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46842 deadline: 1732486619820, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=744bbec6934be304e01b009be760323d, server=bd6592c37bb2,40087,1732486423713 2024-11-24T22:15:59,831 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732486556329.744bbec6934be304e01b009be760323d., hostname=bd6592c37bb2,40087,1732486423713, seqNum=2 , the old value is region=TestAcidGuarantees,,1732486556329.744bbec6934be304e01b009be760323d., hostname=bd6592c37bb2,40087,1732486423713, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=744bbec6934be304e01b009be760323d, server=bd6592c37bb2,40087,1732486423713 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T22:15:59,831 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732486556329.744bbec6934be304e01b009be760323d., hostname=bd6592c37bb2,40087,1732486423713, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=744bbec6934be304e01b009be760323d, server=bd6592c37bb2,40087,1732486423713 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T22:15:59,831 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732486556329.744bbec6934be304e01b009be760323d., hostname=bd6592c37bb2,40087,1732486423713, seqNum=2 because the exception is null or not the one we care about 2024-11-24T22:15:59,841 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40087 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=744bbec6934be304e01b009be760323d, server=bd6592c37bb2,40087,1732486423713 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-24T22:15:59,841 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40087 {}] ipc.CallRunner(138): callId: 14 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46816 deadline: 1732486619831, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=744bbec6934be304e01b009be760323d, server=bd6592c37bb2,40087,1732486423713 2024-11-24T22:15:59,841 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732486556329.744bbec6934be304e01b009be760323d., hostname=bd6592c37bb2,40087,1732486423713, seqNum=2 , the old value is region=TestAcidGuarantees,,1732486556329.744bbec6934be304e01b009be760323d., hostname=bd6592c37bb2,40087,1732486423713, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=744bbec6934be304e01b009be760323d, server=bd6592c37bb2,40087,1732486423713 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T22:15:59,841 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732486556329.744bbec6934be304e01b009be760323d., hostname=bd6592c37bb2,40087,1732486423713, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=744bbec6934be304e01b009be760323d, server=bd6592c37bb2,40087,1732486423713 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T22:15:59,841 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732486556329.744bbec6934be304e01b009be760323d., hostname=bd6592c37bb2,40087,1732486423713, seqNum=2 because the exception is null or not the one we care about 2024-11-24T22:15:59,841 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40087 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=744bbec6934be304e01b009be760323d, server=bd6592c37bb2,40087,1732486423713 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-24T22:15:59,842 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40087 {}] ipc.CallRunner(138): callId: 14 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46824 deadline: 1732486619831, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=744bbec6934be304e01b009be760323d, server=bd6592c37bb2,40087,1732486423713 2024-11-24T22:15:59,842 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732486556329.744bbec6934be304e01b009be760323d., hostname=bd6592c37bb2,40087,1732486423713, seqNum=2 , the old value is region=TestAcidGuarantees,,1732486556329.744bbec6934be304e01b009be760323d., hostname=bd6592c37bb2,40087,1732486423713, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=744bbec6934be304e01b009be760323d, server=bd6592c37bb2,40087,1732486423713 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T22:15:59,842 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40087 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=744bbec6934be304e01b009be760323d, server=bd6592c37bb2,40087,1732486423713 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-24T22:15:59,842 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732486556329.744bbec6934be304e01b009be760323d., hostname=bd6592c37bb2,40087,1732486423713, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=744bbec6934be304e01b009be760323d, server=bd6592c37bb2,40087,1732486423713 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T22:15:59,842 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732486556329.744bbec6934be304e01b009be760323d., hostname=bd6592c37bb2,40087,1732486423713, seqNum=2 because the exception is null or not the one we care about 2024-11-24T22:15:59,842 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40087 {}] ipc.CallRunner(138): callId: 14 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46820 deadline: 1732486619831, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=744bbec6934be304e01b009be760323d, server=bd6592c37bb2,40087,1732486423713 2024-11-24T22:15:59,842 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732486556329.744bbec6934be304e01b009be760323d., hostname=bd6592c37bb2,40087,1732486423713, seqNum=2 , the old value is region=TestAcidGuarantees,,1732486556329.744bbec6934be304e01b009be760323d., hostname=bd6592c37bb2,40087,1732486423713, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=744bbec6934be304e01b009be760323d, server=bd6592c37bb2,40087,1732486423713 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T22:15:59,842 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732486556329.744bbec6934be304e01b009be760323d., hostname=bd6592c37bb2,40087,1732486423713, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=744bbec6934be304e01b009be760323d, server=bd6592c37bb2,40087,1732486423713 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T22:15:59,842 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732486556329.744bbec6934be304e01b009be760323d., hostname=bd6592c37bb2,40087,1732486423713, seqNum=2 because the exception is null or not the one we care about 2024-11-24T22:15:59,859 INFO [RS_FLUSH_OPERATIONS-regionserver/bd6592c37bb2:0-2 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=42.49 KB at sequenceid=47 (bloomFilter=true), to=hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/744bbec6934be304e01b009be760323d/.tmp/B/6d14ee73785241eca44e62a589b062d7 2024-11-24T22:15:59,867 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bd6592c37bb2:0-2 {event_type=RS_FLUSH_REGIONS, pid=122}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/744bbec6934be304e01b009be760323d/.tmp/C/70cb46b606d741fb9cfa26d290b0aac8 is 50, key is test_row_0/C:col10/1732486558630/Put/seqid=0 2024-11-24T22:15:59,870 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45929 is added to blk_1073742283_1459 (size=12001) 2024-11-24T22:15:59,913 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40087 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=744bbec6934be304e01b009be760323d, server=bd6592c37bb2,40087,1732486423713 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-24T22:15:59,913 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40087 {}] ipc.CallRunner(138): callId: 20 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46792 deadline: 1732486619911, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=744bbec6934be304e01b009be760323d, server=bd6592c37bb2,40087,1732486423713 2024-11-24T22:15:59,914 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732486556329.744bbec6934be304e01b009be760323d., hostname=bd6592c37bb2,40087,1732486423713, seqNum=2 , the old value is region=TestAcidGuarantees,,1732486556329.744bbec6934be304e01b009be760323d., hostname=bd6592c37bb2,40087,1732486423713, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=744bbec6934be304e01b009be760323d, server=bd6592c37bb2,40087,1732486423713 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T22:15:59,914 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732486556329.744bbec6934be304e01b009be760323d., hostname=bd6592c37bb2,40087,1732486423713, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=744bbec6934be304e01b009be760323d, server=bd6592c37bb2,40087,1732486423713 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T22:15:59,914 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732486556329.744bbec6934be304e01b009be760323d., hostname=bd6592c37bb2,40087,1732486423713, seqNum=2 because the exception is null or not the one we care about 2024-11-24T22:16:00,131 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40087 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=744bbec6934be304e01b009be760323d, server=bd6592c37bb2,40087,1732486423713 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-24T22:16:00,131 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40087 {}] ipc.CallRunner(138): callId: 21 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46792 deadline: 1732486620120, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=744bbec6934be304e01b009be760323d, server=bd6592c37bb2,40087,1732486423713 2024-11-24T22:16:00,131 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732486556329.744bbec6934be304e01b009be760323d., hostname=bd6592c37bb2,40087,1732486423713, seqNum=2 , the old value is region=TestAcidGuarantees,,1732486556329.744bbec6934be304e01b009be760323d., hostname=bd6592c37bb2,40087,1732486423713, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=744bbec6934be304e01b009be760323d, server=bd6592c37bb2,40087,1732486423713 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T22:16:00,131 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732486556329.744bbec6934be304e01b009be760323d., hostname=bd6592c37bb2,40087,1732486423713, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=744bbec6934be304e01b009be760323d, server=bd6592c37bb2,40087,1732486423713 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T22:16:00,131 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732486556329.744bbec6934be304e01b009be760323d., hostname=bd6592c37bb2,40087,1732486423713, seqNum=2 because the exception is null or not the one we care about 2024-11-24T22:16:00,271 INFO [RS_FLUSH_OPERATIONS-regionserver/bd6592c37bb2:0-2 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=42.49 KB at sequenceid=47 (bloomFilter=true), to=hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/744bbec6934be304e01b009be760323d/.tmp/C/70cb46b606d741fb9cfa26d290b0aac8 2024-11-24T22:16:00,275 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bd6592c37bb2:0-2 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/744bbec6934be304e01b009be760323d/.tmp/A/9338a57d248a4d55bbcb179d0bffa465 as hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/744bbec6934be304e01b009be760323d/A/9338a57d248a4d55bbcb179d0bffa465 2024-11-24T22:16:00,282 INFO [RS_FLUSH_OPERATIONS-regionserver/bd6592c37bb2:0-2 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/744bbec6934be304e01b009be760323d/A/9338a57d248a4d55bbcb179d0bffa465, entries=150, sequenceid=47, filesize=11.7 K 2024-11-24T22:16:00,282 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bd6592c37bb2:0-2 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/744bbec6934be304e01b009be760323d/.tmp/B/6d14ee73785241eca44e62a589b062d7 as hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/744bbec6934be304e01b009be760323d/B/6d14ee73785241eca44e62a589b062d7 2024-11-24T22:16:00,286 INFO [RS_FLUSH_OPERATIONS-regionserver/bd6592c37bb2:0-2 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/744bbec6934be304e01b009be760323d/B/6d14ee73785241eca44e62a589b062d7, entries=150, sequenceid=47, filesize=11.7 K 2024-11-24T22:16:00,286 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bd6592c37bb2:0-2 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/744bbec6934be304e01b009be760323d/.tmp/C/70cb46b606d741fb9cfa26d290b0aac8 as hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/744bbec6934be304e01b009be760323d/C/70cb46b606d741fb9cfa26d290b0aac8 2024-11-24T22:16:00,290 INFO [RS_FLUSH_OPERATIONS-regionserver/bd6592c37bb2:0-2 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/744bbec6934be304e01b009be760323d/C/70cb46b606d741fb9cfa26d290b0aac8, entries=150, sequenceid=47, filesize=11.7 K 2024-11-24T22:16:00,291 INFO [RS_FLUSH_OPERATIONS-regionserver/bd6592c37bb2:0-2 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.HRegion(3140): Finished flush of dataSize ~127.47 KB/130530, heapSize ~360.33 KB/368976, currentSize=53.67 KB/54960 for 744bbec6934be304e01b009be760323d in 853ms, sequenceid=47, compaction requested=false 2024-11-24T22:16:00,291 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bd6592c37bb2:0-2 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.HRegion(2603): Flush status journal for 744bbec6934be304e01b009be760323d: 2024-11-24T22:16:00,291 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bd6592c37bb2:0-2 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732486556329.744bbec6934be304e01b009be760323d. 2024-11-24T22:16:00,291 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bd6592c37bb2:0-2 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=122 2024-11-24T22:16:00,292 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37701 {}] master.HMaster(4169): Remote procedure done, pid=122 2024-11-24T22:16:00,295 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=122, resume processing ppid=121 2024-11-24T22:16:00,295 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=122, ppid=121, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.7730 sec 2024-11-24T22:16:00,299 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=121, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=121, table=TestAcidGuarantees in 1.7800 sec 2024-11-24T22:16:00,445 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40087 {}] regionserver.HRegion(8855): Flush requested on 744bbec6934be304e01b009be760323d 2024-11-24T22:16:00,445 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 744bbec6934be304e01b009be760323d 3/3 column families, dataSize=60.38 KB heapSize=180.05 KB 2024-11-24T22:16:00,445 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 744bbec6934be304e01b009be760323d, store=A 2024-11-24T22:16:00,445 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-24T22:16:00,445 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 744bbec6934be304e01b009be760323d, store=B 2024-11-24T22:16:00,446 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-24T22:16:00,446 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 744bbec6934be304e01b009be760323d, store=C 2024-11-24T22:16:00,446 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-24T22:16:00,450 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/744bbec6934be304e01b009be760323d/.tmp/A/87d7051b03c6450b9211666fb515ffe9 is 50, key is test_row_0/A:col10/1732486559779/Put/seqid=0 2024-11-24T22:16:00,454 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45929 is added to blk_1073742284_1460 (size=14341) 2024-11-24T22:16:00,576 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40087 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=744bbec6934be304e01b009be760323d, server=bd6592c37bb2,40087,1732486423713 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-24T22:16:00,576 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40087 {}] ipc.CallRunner(138): callId: 40 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46792 deadline: 1732486620563, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=744bbec6934be304e01b009be760323d, server=bd6592c37bb2,40087,1732486423713 2024-11-24T22:16:00,576 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732486556329.744bbec6934be304e01b009be760323d., hostname=bd6592c37bb2,40087,1732486423713, seqNum=2 , the old value is region=TestAcidGuarantees,,1732486556329.744bbec6934be304e01b009be760323d., hostname=bd6592c37bb2,40087,1732486423713, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=744bbec6934be304e01b009be760323d, server=bd6592c37bb2,40087,1732486423713 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T22:16:00,576 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732486556329.744bbec6934be304e01b009be760323d., hostname=bd6592c37bb2,40087,1732486423713, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=744bbec6934be304e01b009be760323d, server=bd6592c37bb2,40087,1732486423713 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T22:16:00,576 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732486556329.744bbec6934be304e01b009be760323d., hostname=bd6592c37bb2,40087,1732486423713, seqNum=2 because the exception is null or not the one we care about 2024-11-24T22:16:00,660 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37701 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=121 2024-11-24T22:16:00,661 INFO [RPCClient-NioEventLoopGroup-4-10 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-24T22:16:00,661 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37701 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-24T22:16:00,662 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37701 {}] procedure2.ProcedureExecutor(1139): Stored pid=123, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=123, table=TestAcidGuarantees 2024-11-24T22:16:00,662 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37701 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=123 2024-11-24T22:16:00,663 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=123, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=123, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-24T22:16:00,663 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=123, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=123, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-24T22:16:00,663 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=124, ppid=123, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-24T22:16:00,691 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40087 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=744bbec6934be304e01b009be760323d, server=bd6592c37bb2,40087,1732486423713 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-24T22:16:00,691 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40087 {}] ipc.CallRunner(138): callId: 41 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46792 deadline: 1732486620681, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=744bbec6934be304e01b009be760323d, server=bd6592c37bb2,40087,1732486423713 2024-11-24T22:16:00,692 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732486556329.744bbec6934be304e01b009be760323d., hostname=bd6592c37bb2,40087,1732486423713, seqNum=2 , the old value is region=TestAcidGuarantees,,1732486556329.744bbec6934be304e01b009be760323d., hostname=bd6592c37bb2,40087,1732486423713, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=744bbec6934be304e01b009be760323d, server=bd6592c37bb2,40087,1732486423713 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T22:16:00,692 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732486556329.744bbec6934be304e01b009be760323d., hostname=bd6592c37bb2,40087,1732486423713, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=744bbec6934be304e01b009be760323d, server=bd6592c37bb2,40087,1732486423713 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T22:16:00,692 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732486556329.744bbec6934be304e01b009be760323d., hostname=bd6592c37bb2,40087,1732486423713, seqNum=2 because the exception is null or not the one we care about 2024-11-24T22:16:00,770 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37701 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=123 2024-11-24T22:16:00,816 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=40087 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=124 2024-11-24T22:16:00,816 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bd6592c37bb2:0-0 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732486556329.744bbec6934be304e01b009be760323d. 2024-11-24T22:16:00,816 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bd6592c37bb2:0-0 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732486556329.744bbec6934be304e01b009be760323d. as already flushing 2024-11-24T22:16:00,816 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bd6592c37bb2:0-0 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732486556329.744bbec6934be304e01b009be760323d. 2024-11-24T22:16:00,816 ERROR [RS_FLUSH_OPERATIONS-regionserver/bd6592c37bb2:0-0 {event_type=RS_FLUSH_REGIONS, pid=124}] handler.RSProcedureHandler(58): pid=124 java.io.IOException: Unable to complete flush {ENCODED => 744bbec6934be304e01b009be760323d, NAME => 'TestAcidGuarantees,,1732486556329.744bbec6934be304e01b009be760323d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T22:16:00,816 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bd6592c37bb2:0-0 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=124 java.io.IOException: Unable to complete flush {ENCODED => 744bbec6934be304e01b009be760323d, NAME => 'TestAcidGuarantees,,1732486556329.744bbec6934be304e01b009be760323d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T22:16:00,817 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37701 {}] master.HMaster(4177): Remote procedure failed, pid=124 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 744bbec6934be304e01b009be760323d, NAME => 'TestAcidGuarantees,,1732486556329.744bbec6934be304e01b009be760323d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 744bbec6934be304e01b009be760323d, NAME => 'TestAcidGuarantees,,1732486556329.744bbec6934be304e01b009be760323d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T22:16:00,848 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40087 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=744bbec6934be304e01b009be760323d, server=bd6592c37bb2,40087,1732486423713 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-24T22:16:00,849 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40087 {}] ipc.CallRunner(138): callId: 13 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46842 deadline: 1732486620841, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=744bbec6934be304e01b009be760323d, server=bd6592c37bb2,40087,1732486423713 2024-11-24T22:16:00,849 WARN [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRpcRetryingCaller(177): Call to bd6592c37bb2,40087,1732486423713 for 'test_row_1' in 744bbec6934be304e01b009be760323d of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2203 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=744bbec6934be304e01b009be760323d, server=bd6592c37bb2,40087,1732486423713 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor44.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=744bbec6934be304e01b009be760323d, server=bd6592c37bb2,40087,1732486423713 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-24T22:16:00,850 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732486556329.744bbec6934be304e01b009be760323d., hostname=bd6592c37bb2,40087,1732486423713, seqNum=2 , the old value is region=TestAcidGuarantees,,1732486556329.744bbec6934be304e01b009be760323d., hostname=bd6592c37bb2,40087,1732486423713, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=744bbec6934be304e01b009be760323d, server=bd6592c37bb2,40087,1732486423713 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T22:16:00,850 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732486556329.744bbec6934be304e01b009be760323d., hostname=bd6592c37bb2,40087,1732486423713, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=744bbec6934be304e01b009be760323d, server=bd6592c37bb2,40087,1732486423713 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T22:16:00,850 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732486556329.744bbec6934be304e01b009be760323d., hostname=bd6592c37bb2,40087,1732486423713, seqNum=2 because the exception is null or not the one we care about 2024-11-24T22:16:00,855 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=59 (bloomFilter=true), to=hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/744bbec6934be304e01b009be760323d/.tmp/A/87d7051b03c6450b9211666fb515ffe9 2024-11-24T22:16:00,858 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40087 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=744bbec6934be304e01b009be760323d, server=bd6592c37bb2,40087,1732486423713 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-24T22:16:00,858 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40087 {}] ipc.CallRunner(138): callId: 15 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46824 deadline: 1732486620850, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=744bbec6934be304e01b009be760323d, server=bd6592c37bb2,40087,1732486423713 2024-11-24T22:16:00,858 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40087 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=744bbec6934be304e01b009be760323d, server=bd6592c37bb2,40087,1732486423713 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-24T22:16:00,858 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40087 {}] ipc.CallRunner(138): callId: 15 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46820 deadline: 1732486620851, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=744bbec6934be304e01b009be760323d, server=bd6592c37bb2,40087,1732486423713 2024-11-24T22:16:00,858 WARN [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRpcRetryingCaller(177): Call to bd6592c37bb2,40087,1732486423713 for 'test_row_1' in 744bbec6934be304e01b009be760323d of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2212 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=744bbec6934be304e01b009be760323d, server=bd6592c37bb2,40087,1732486423713 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor44.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=744bbec6934be304e01b009be760323d, server=bd6592c37bb2,40087,1732486423713 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-24T22:16:00,859 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732486556329.744bbec6934be304e01b009be760323d., hostname=bd6592c37bb2,40087,1732486423713, seqNum=2 , the old value is region=TestAcidGuarantees,,1732486556329.744bbec6934be304e01b009be760323d., hostname=bd6592c37bb2,40087,1732486423713, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=744bbec6934be304e01b009be760323d, server=bd6592c37bb2,40087,1732486423713 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T22:16:00,858 WARN [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRpcRetryingCaller(177): Call to bd6592c37bb2,40087,1732486423713 for 'test_row_2' in 744bbec6934be304e01b009be760323d of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2209 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=744bbec6934be304e01b009be760323d, server=bd6592c37bb2,40087,1732486423713 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor44.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=744bbec6934be304e01b009be760323d, server=bd6592c37bb2,40087,1732486423713 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-24T22:16:00,859 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732486556329.744bbec6934be304e01b009be760323d., hostname=bd6592c37bb2,40087,1732486423713, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=744bbec6934be304e01b009be760323d, server=bd6592c37bb2,40087,1732486423713 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T22:16:00,859 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732486556329.744bbec6934be304e01b009be760323d., hostname=bd6592c37bb2,40087,1732486423713, seqNum=2 because the exception is null or not the one we care about 2024-11-24T22:16:00,859 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732486556329.744bbec6934be304e01b009be760323d., hostname=bd6592c37bb2,40087,1732486423713, seqNum=2 , the old value is region=TestAcidGuarantees,,1732486556329.744bbec6934be304e01b009be760323d., hostname=bd6592c37bb2,40087,1732486423713, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=744bbec6934be304e01b009be760323d, server=bd6592c37bb2,40087,1732486423713 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T22:16:00,859 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732486556329.744bbec6934be304e01b009be760323d., hostname=bd6592c37bb2,40087,1732486423713, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=744bbec6934be304e01b009be760323d, server=bd6592c37bb2,40087,1732486423713 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T22:16:00,859 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732486556329.744bbec6934be304e01b009be760323d., hostname=bd6592c37bb2,40087,1732486423713, seqNum=2 because the exception is null or not the one we care about 2024-11-24T22:16:00,862 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/744bbec6934be304e01b009be760323d/.tmp/B/53ccdbef1ae447f1868f5cf9ab39d65e is 50, key is test_row_0/B:col10/1732486559779/Put/seqid=0 2024-11-24T22:16:00,865 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45929 is added to blk_1073742285_1461 (size=12001) 2024-11-24T22:16:00,872 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40087 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=744bbec6934be304e01b009be760323d, server=bd6592c37bb2,40087,1732486423713 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-24T22:16:00,872 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40087 {}] ipc.CallRunner(138): callId: 15 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46816 deadline: 1732486620861, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=744bbec6934be304e01b009be760323d, server=bd6592c37bb2,40087,1732486423713 2024-11-24T22:16:00,872 WARN [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRpcRetryingCaller(177): Call to bd6592c37bb2,40087,1732486423713 for 'test_row_1' in 744bbec6934be304e01b009be760323d of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2228 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=744bbec6934be304e01b009be760323d, server=bd6592c37bb2,40087,1732486423713 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor44.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=744bbec6934be304e01b009be760323d, server=bd6592c37bb2,40087,1732486423713 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-24T22:16:00,873 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732486556329.744bbec6934be304e01b009be760323d., hostname=bd6592c37bb2,40087,1732486423713, seqNum=2 , the old value is region=TestAcidGuarantees,,1732486556329.744bbec6934be304e01b009be760323d., hostname=bd6592c37bb2,40087,1732486423713, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=744bbec6934be304e01b009be760323d, server=bd6592c37bb2,40087,1732486423713 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T22:16:00,873 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732486556329.744bbec6934be304e01b009be760323d., hostname=bd6592c37bb2,40087,1732486423713, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=744bbec6934be304e01b009be760323d, server=bd6592c37bb2,40087,1732486423713 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T22:16:00,873 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732486556329.744bbec6934be304e01b009be760323d., hostname=bd6592c37bb2,40087,1732486423713, seqNum=2 because the exception is null or not the one we care about 2024-11-24T22:16:00,913 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40087 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=744bbec6934be304e01b009be760323d, server=bd6592c37bb2,40087,1732486423713 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-24T22:16:00,914 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40087 {}] ipc.CallRunner(138): callId: 42 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46792 deadline: 1732486620900, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=744bbec6934be304e01b009be760323d, server=bd6592c37bb2,40087,1732486423713 2024-11-24T22:16:00,914 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732486556329.744bbec6934be304e01b009be760323d., hostname=bd6592c37bb2,40087,1732486423713, seqNum=2 , the old value is region=TestAcidGuarantees,,1732486556329.744bbec6934be304e01b009be760323d., hostname=bd6592c37bb2,40087,1732486423713, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=744bbec6934be304e01b009be760323d, server=bd6592c37bb2,40087,1732486423713 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T22:16:00,914 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732486556329.744bbec6934be304e01b009be760323d., hostname=bd6592c37bb2,40087,1732486423713, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=744bbec6934be304e01b009be760323d, server=bd6592c37bb2,40087,1732486423713 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T22:16:00,914 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732486556329.744bbec6934be304e01b009be760323d., hostname=bd6592c37bb2,40087,1732486423713, seqNum=2 because the exception is null or not the one we care about 2024-11-24T22:16:00,969 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=40087 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=124 2024-11-24T22:16:00,969 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bd6592c37bb2:0-1 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732486556329.744bbec6934be304e01b009be760323d. 2024-11-24T22:16:00,969 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bd6592c37bb2:0-1 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732486556329.744bbec6934be304e01b009be760323d. as already flushing 2024-11-24T22:16:00,969 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bd6592c37bb2:0-1 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732486556329.744bbec6934be304e01b009be760323d. 2024-11-24T22:16:00,969 ERROR [RS_FLUSH_OPERATIONS-regionserver/bd6592c37bb2:0-1 {event_type=RS_FLUSH_REGIONS, pid=124}] handler.RSProcedureHandler(58): pid=124 java.io.IOException: Unable to complete flush {ENCODED => 744bbec6934be304e01b009be760323d, NAME => 'TestAcidGuarantees,,1732486556329.744bbec6934be304e01b009be760323d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T22:16:00,970 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bd6592c37bb2:0-1 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=124 java.io.IOException: Unable to complete flush {ENCODED => 744bbec6934be304e01b009be760323d, NAME => 'TestAcidGuarantees,,1732486556329.744bbec6934be304e01b009be760323d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T22:16:00,970 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37701 {}] master.HMaster(4177): Remote procedure failed, pid=124 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 744bbec6934be304e01b009be760323d, NAME => 'TestAcidGuarantees,,1732486556329.744bbec6934be304e01b009be760323d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 744bbec6934be304e01b009be760323d, NAME => 'TestAcidGuarantees,,1732486556329.744bbec6934be304e01b009be760323d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T22:16:00,981 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37701 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=123 2024-11-24T22:16:01,122 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=40087 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=124 2024-11-24T22:16:01,122 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bd6592c37bb2:0-2 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732486556329.744bbec6934be304e01b009be760323d. 2024-11-24T22:16:01,123 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bd6592c37bb2:0-2 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732486556329.744bbec6934be304e01b009be760323d. as already flushing 2024-11-24T22:16:01,123 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bd6592c37bb2:0-2 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732486556329.744bbec6934be304e01b009be760323d. 2024-11-24T22:16:01,123 ERROR [RS_FLUSH_OPERATIONS-regionserver/bd6592c37bb2:0-2 {event_type=RS_FLUSH_REGIONS, pid=124}] handler.RSProcedureHandler(58): pid=124 java.io.IOException: Unable to complete flush {ENCODED => 744bbec6934be304e01b009be760323d, NAME => 'TestAcidGuarantees,,1732486556329.744bbec6934be304e01b009be760323d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T22:16:01,123 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bd6592c37bb2:0-2 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=124 java.io.IOException: Unable to complete flush {ENCODED => 744bbec6934be304e01b009be760323d, NAME => 'TestAcidGuarantees,,1732486556329.744bbec6934be304e01b009be760323d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T22:16:01,123 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37701 {}] master.HMaster(4177): Remote procedure failed, pid=124 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 744bbec6934be304e01b009be760323d, NAME => 'TestAcidGuarantees,,1732486556329.744bbec6934be304e01b009be760323d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 744bbec6934be304e01b009be760323d, NAME => 'TestAcidGuarantees,,1732486556329.744bbec6934be304e01b009be760323d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T22:16:01,227 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40087 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=744bbec6934be304e01b009be760323d, server=bd6592c37bb2,40087,1732486423713 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-24T22:16:01,227 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40087 {}] ipc.CallRunner(138): callId: 43 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46792 deadline: 1732486621221, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=744bbec6934be304e01b009be760323d, server=bd6592c37bb2,40087,1732486423713 2024-11-24T22:16:01,227 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732486556329.744bbec6934be304e01b009be760323d., hostname=bd6592c37bb2,40087,1732486423713, seqNum=2 , the old value is region=TestAcidGuarantees,,1732486556329.744bbec6934be304e01b009be760323d., hostname=bd6592c37bb2,40087,1732486423713, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=744bbec6934be304e01b009be760323d, server=bd6592c37bb2,40087,1732486423713 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T22:16:01,227 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732486556329.744bbec6934be304e01b009be760323d., hostname=bd6592c37bb2,40087,1732486423713, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=744bbec6934be304e01b009be760323d, server=bd6592c37bb2,40087,1732486423713 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T22:16:01,227 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732486556329.744bbec6934be304e01b009be760323d., hostname=bd6592c37bb2,40087,1732486423713, seqNum=2 because the exception is null or not the one we care about 2024-11-24T22:16:01,265 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=59 (bloomFilter=true), to=hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/744bbec6934be304e01b009be760323d/.tmp/B/53ccdbef1ae447f1868f5cf9ab39d65e 2024-11-24T22:16:01,273 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/744bbec6934be304e01b009be760323d/.tmp/C/33440ac6cc804bad90dd4f9ef9db6daa is 50, key is test_row_0/C:col10/1732486559779/Put/seqid=0 2024-11-24T22:16:01,275 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=40087 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=124 2024-11-24T22:16:01,275 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bd6592c37bb2:0-0 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732486556329.744bbec6934be304e01b009be760323d. 2024-11-24T22:16:01,276 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bd6592c37bb2:0-0 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732486556329.744bbec6934be304e01b009be760323d. as already flushing 2024-11-24T22:16:01,276 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bd6592c37bb2:0-0 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732486556329.744bbec6934be304e01b009be760323d. 2024-11-24T22:16:01,276 ERROR [RS_FLUSH_OPERATIONS-regionserver/bd6592c37bb2:0-0 {event_type=RS_FLUSH_REGIONS, pid=124}] handler.RSProcedureHandler(58): pid=124 java.io.IOException: Unable to complete flush {ENCODED => 744bbec6934be304e01b009be760323d, NAME => 'TestAcidGuarantees,,1732486556329.744bbec6934be304e01b009be760323d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T22:16:01,276 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bd6592c37bb2:0-0 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=124 java.io.IOException: Unable to complete flush {ENCODED => 744bbec6934be304e01b009be760323d, NAME => 'TestAcidGuarantees,,1732486556329.744bbec6934be304e01b009be760323d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T22:16:01,276 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37701 {}] master.HMaster(4177): Remote procedure failed, pid=124 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 744bbec6934be304e01b009be760323d, NAME => 'TestAcidGuarantees,,1732486556329.744bbec6934be304e01b009be760323d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 744bbec6934be304e01b009be760323d, NAME => 'TestAcidGuarantees,,1732486556329.744bbec6934be304e01b009be760323d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T22:16:01,286 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45929 is added to blk_1073742286_1462 (size=12001) 2024-11-24T22:16:01,287 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=59 (bloomFilter=true), to=hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/744bbec6934be304e01b009be760323d/.tmp/C/33440ac6cc804bad90dd4f9ef9db6daa 2024-11-24T22:16:01,290 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37701 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=123 2024-11-24T22:16:01,290 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/744bbec6934be304e01b009be760323d/.tmp/A/87d7051b03c6450b9211666fb515ffe9 as hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/744bbec6934be304e01b009be760323d/A/87d7051b03c6450b9211666fb515ffe9 2024-11-24T22:16:01,297 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/744bbec6934be304e01b009be760323d/A/87d7051b03c6450b9211666fb515ffe9, entries=200, sequenceid=59, filesize=14.0 K 2024-11-24T22:16:01,297 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/744bbec6934be304e01b009be760323d/.tmp/B/53ccdbef1ae447f1868f5cf9ab39d65e as hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/744bbec6934be304e01b009be760323d/B/53ccdbef1ae447f1868f5cf9ab39d65e 2024-11-24T22:16:01,304 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/744bbec6934be304e01b009be760323d/B/53ccdbef1ae447f1868f5cf9ab39d65e, entries=150, sequenceid=59, filesize=11.7 K 2024-11-24T22:16:01,304 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/744bbec6934be304e01b009be760323d/.tmp/C/33440ac6cc804bad90dd4f9ef9db6daa as hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/744bbec6934be304e01b009be760323d/C/33440ac6cc804bad90dd4f9ef9db6daa 2024-11-24T22:16:01,310 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/744bbec6934be304e01b009be760323d/C/33440ac6cc804bad90dd4f9ef9db6daa, entries=150, sequenceid=59, filesize=11.7 K 2024-11-24T22:16:01,311 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~60.38 KB/61830, heapSize ~180 KB/184320, currentSize=114.05 KB/116790 for 744bbec6934be304e01b009be760323d in 866ms, sequenceid=59, compaction requested=true 2024-11-24T22:16:01,311 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 744bbec6934be304e01b009be760323d: 2024-11-24T22:16:01,311 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 744bbec6934be304e01b009be760323d:A, priority=-2147483648, current under compaction store size is 1 2024-11-24T22:16:01,311 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-24T22:16:01,311 DEBUG [RS:0;bd6592c37bb2:40087-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-24T22:16:01,311 DEBUG [RS:0;bd6592c37bb2:40087-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-24T22:16:01,311 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 744bbec6934be304e01b009be760323d:B, priority=-2147483648, current under compaction store size is 2 2024-11-24T22:16:01,311 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-24T22:16:01,311 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 744bbec6934be304e01b009be760323d:C, priority=-2147483648, current under compaction store size is 3 2024-11-24T22:16:01,311 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-24T22:16:01,312 DEBUG [RS:0;bd6592c37bb2:40087-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36003 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-24T22:16:01,312 DEBUG [RS:0;bd6592c37bb2:40087-longCompactions-0 {}] regionserver.HStore(1541): 744bbec6934be304e01b009be760323d/B is initiating minor compaction (all files) 2024-11-24T22:16:01,312 INFO [RS:0;bd6592c37bb2:40087-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 744bbec6934be304e01b009be760323d/B in TestAcidGuarantees,,1732486556329.744bbec6934be304e01b009be760323d. 2024-11-24T22:16:01,312 INFO [RS:0;bd6592c37bb2:40087-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/744bbec6934be304e01b009be760323d/B/6f1957bed3164d5aa2811fb5eb436e11, hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/744bbec6934be304e01b009be760323d/B/6d14ee73785241eca44e62a589b062d7, hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/744bbec6934be304e01b009be760323d/B/53ccdbef1ae447f1868f5cf9ab39d65e] into tmpdir=hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/744bbec6934be304e01b009be760323d/.tmp, totalSize=35.2 K 2024-11-24T22:16:01,313 DEBUG [RS:0;bd6592c37bb2:40087-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 40683 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-24T22:16:01,313 DEBUG [RS:0;bd6592c37bb2:40087-shortCompactions-0 {}] regionserver.HStore(1541): 744bbec6934be304e01b009be760323d/A is initiating minor compaction (all files) 2024-11-24T22:16:01,313 INFO [RS:0;bd6592c37bb2:40087-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 744bbec6934be304e01b009be760323d/A in TestAcidGuarantees,,1732486556329.744bbec6934be304e01b009be760323d. 2024-11-24T22:16:01,313 INFO [RS:0;bd6592c37bb2:40087-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/744bbec6934be304e01b009be760323d/A/56e7f0a11f784f479c172eeabafc39b7, hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/744bbec6934be304e01b009be760323d/A/9338a57d248a4d55bbcb179d0bffa465, hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/744bbec6934be304e01b009be760323d/A/87d7051b03c6450b9211666fb515ffe9] into tmpdir=hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/744bbec6934be304e01b009be760323d/.tmp, totalSize=39.7 K 2024-11-24T22:16:01,313 DEBUG [RS:0;bd6592c37bb2:40087-shortCompactions-0 {}] compactions.Compactor(225): Compacting 56e7f0a11f784f479c172eeabafc39b7, keycount=200, bloomtype=ROW, size=14.0 K, encoding=NONE, compression=NONE, seqNum=12, earliestPutTs=1732486558524 2024-11-24T22:16:01,313 DEBUG [RS:0;bd6592c37bb2:40087-longCompactions-0 {}] compactions.Compactor(225): Compacting 6f1957bed3164d5aa2811fb5eb436e11, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=12, earliestPutTs=1732486558524 2024-11-24T22:16:01,313 DEBUG [RS:0;bd6592c37bb2:40087-shortCompactions-0 {}] compactions.Compactor(225): Compacting 9338a57d248a4d55bbcb179d0bffa465, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=47, earliestPutTs=1732486558630 2024-11-24T22:16:01,314 DEBUG [RS:0;bd6592c37bb2:40087-longCompactions-0 {}] compactions.Compactor(225): Compacting 6d14ee73785241eca44e62a589b062d7, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=47, earliestPutTs=1732486558630 2024-11-24T22:16:01,314 DEBUG [RS:0;bd6592c37bb2:40087-shortCompactions-0 {}] compactions.Compactor(225): Compacting 87d7051b03c6450b9211666fb515ffe9, keycount=200, bloomtype=ROW, size=14.0 K, encoding=NONE, compression=NONE, seqNum=59, earliestPutTs=1732486559779 2024-11-24T22:16:01,314 DEBUG [RS:0;bd6592c37bb2:40087-longCompactions-0 {}] compactions.Compactor(225): Compacting 53ccdbef1ae447f1868f5cf9ab39d65e, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=59, earliestPutTs=1732486559779 2024-11-24T22:16:01,327 INFO [RS:0;bd6592c37bb2:40087-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 744bbec6934be304e01b009be760323d#A#compaction#383 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 57.69 MB/second 2024-11-24T22:16:01,328 DEBUG [RS:0;bd6592c37bb2:40087-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/744bbec6934be304e01b009be760323d/.tmp/A/6a62440767db4e10be88de20dd521d76 is 50, key is test_row_0/A:col10/1732486559779/Put/seqid=0 2024-11-24T22:16:01,332 INFO [RS:0;bd6592c37bb2:40087-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 744bbec6934be304e01b009be760323d#B#compaction#384 average throughput is 1.64 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 57.69 MB/second 2024-11-24T22:16:01,332 DEBUG [RS:0;bd6592c37bb2:40087-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/744bbec6934be304e01b009be760323d/.tmp/B/1a3fc19c62624ad0a67ac686e829085c is 50, key is test_row_0/B:col10/1732486559779/Put/seqid=0 2024-11-24T22:16:01,342 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45929 is added to blk_1073742287_1463 (size=12104) 2024-11-24T22:16:01,349 DEBUG [RS:0;bd6592c37bb2:40087-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/744bbec6934be304e01b009be760323d/.tmp/A/6a62440767db4e10be88de20dd521d76 as hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/744bbec6934be304e01b009be760323d/A/6a62440767db4e10be88de20dd521d76 2024-11-24T22:16:01,358 INFO [RS:0;bd6592c37bb2:40087-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 744bbec6934be304e01b009be760323d/A of 744bbec6934be304e01b009be760323d into 6a62440767db4e10be88de20dd521d76(size=11.8 K), total size for store is 11.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-24T22:16:01,358 DEBUG [RS:0;bd6592c37bb2:40087-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 744bbec6934be304e01b009be760323d: 2024-11-24T22:16:01,358 INFO [RS:0;bd6592c37bb2:40087-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732486556329.744bbec6934be304e01b009be760323d., storeName=744bbec6934be304e01b009be760323d/A, priority=13, startTime=1732486561311; duration=0sec 2024-11-24T22:16:01,358 DEBUG [RS:0;bd6592c37bb2:40087-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-24T22:16:01,358 DEBUG [RS:0;bd6592c37bb2:40087-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 744bbec6934be304e01b009be760323d:A 2024-11-24T22:16:01,358 DEBUG [RS:0;bd6592c37bb2:40087-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-24T22:16:01,359 DEBUG [RS:0;bd6592c37bb2:40087-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36003 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-24T22:16:01,359 DEBUG [RS:0;bd6592c37bb2:40087-shortCompactions-0 {}] regionserver.HStore(1541): 744bbec6934be304e01b009be760323d/C is initiating minor compaction (all files) 2024-11-24T22:16:01,359 INFO [RS:0;bd6592c37bb2:40087-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 744bbec6934be304e01b009be760323d/C in TestAcidGuarantees,,1732486556329.744bbec6934be304e01b009be760323d. 2024-11-24T22:16:01,359 INFO [RS:0;bd6592c37bb2:40087-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/744bbec6934be304e01b009be760323d/C/d9a1c99c3bfb401092039d1293ad1252, hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/744bbec6934be304e01b009be760323d/C/70cb46b606d741fb9cfa26d290b0aac8, hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/744bbec6934be304e01b009be760323d/C/33440ac6cc804bad90dd4f9ef9db6daa] into tmpdir=hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/744bbec6934be304e01b009be760323d/.tmp, totalSize=35.2 K 2024-11-24T22:16:01,360 DEBUG [RS:0;bd6592c37bb2:40087-shortCompactions-0 {}] compactions.Compactor(225): Compacting d9a1c99c3bfb401092039d1293ad1252, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=12, earliestPutTs=1732486558524 2024-11-24T22:16:01,360 DEBUG [RS:0;bd6592c37bb2:40087-shortCompactions-0 {}] compactions.Compactor(225): Compacting 70cb46b606d741fb9cfa26d290b0aac8, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=47, earliestPutTs=1732486558630 2024-11-24T22:16:01,360 DEBUG [RS:0;bd6592c37bb2:40087-shortCompactions-0 {}] compactions.Compactor(225): Compacting 33440ac6cc804bad90dd4f9ef9db6daa, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=59, earliestPutTs=1732486559779 2024-11-24T22:16:01,366 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45929 is added to blk_1073742288_1464 (size=12104) 2024-11-24T22:16:01,373 DEBUG [RS:0;bd6592c37bb2:40087-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/744bbec6934be304e01b009be760323d/.tmp/B/1a3fc19c62624ad0a67ac686e829085c as hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/744bbec6934be304e01b009be760323d/B/1a3fc19c62624ad0a67ac686e829085c 2024-11-24T22:16:01,377 INFO [RS:0;bd6592c37bb2:40087-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 744bbec6934be304e01b009be760323d/B of 744bbec6934be304e01b009be760323d into 1a3fc19c62624ad0a67ac686e829085c(size=11.8 K), total size for store is 11.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-24T22:16:01,377 DEBUG [RS:0;bd6592c37bb2:40087-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 744bbec6934be304e01b009be760323d: 2024-11-24T22:16:01,377 INFO [RS:0;bd6592c37bb2:40087-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732486556329.744bbec6934be304e01b009be760323d., storeName=744bbec6934be304e01b009be760323d/B, priority=13, startTime=1732486561311; duration=0sec 2024-11-24T22:16:01,378 DEBUG [RS:0;bd6592c37bb2:40087-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-24T22:16:01,378 DEBUG [RS:0;bd6592c37bb2:40087-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 744bbec6934be304e01b009be760323d:B 2024-11-24T22:16:01,381 INFO [RS:0;bd6592c37bb2:40087-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 744bbec6934be304e01b009be760323d#C#compaction#385 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 57.69 MB/second 2024-11-24T22:16:01,382 DEBUG [RS:0;bd6592c37bb2:40087-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/744bbec6934be304e01b009be760323d/.tmp/C/57a5000178934a4c98dbc71d7e5ffb12 is 50, key is test_row_0/C:col10/1732486559779/Put/seqid=0 2024-11-24T22:16:01,389 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45929 is added to blk_1073742289_1465 (size=12104) 2024-11-24T22:16:01,411 WARN [HBase-Metrics2-1 {}] impl.MetricsConfig(138): Cannot locate configuration: tried hadoop-metrics2-hbase.properties,hadoop-metrics2.properties 2024-11-24T22:16:01,429 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=40087 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=124 2024-11-24T22:16:01,429 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bd6592c37bb2:0-1 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732486556329.744bbec6934be304e01b009be760323d. 2024-11-24T22:16:01,429 INFO [RS_FLUSH_OPERATIONS-regionserver/bd6592c37bb2:0-1 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.HRegion(2902): Flushing 744bbec6934be304e01b009be760323d 3/3 column families, dataSize=114.05 KB heapSize=339.42 KB 2024-11-24T22:16:01,429 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bd6592c37bb2:0-1 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 744bbec6934be304e01b009be760323d, store=A 2024-11-24T22:16:01,429 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bd6592c37bb2:0-1 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-24T22:16:01,429 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bd6592c37bb2:0-1 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 744bbec6934be304e01b009be760323d, store=B 2024-11-24T22:16:01,429 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bd6592c37bb2:0-1 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-24T22:16:01,429 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bd6592c37bb2:0-1 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 744bbec6934be304e01b009be760323d, store=C 2024-11-24T22:16:01,430 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bd6592c37bb2:0-1 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-24T22:16:01,435 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bd6592c37bb2:0-1 {event_type=RS_FLUSH_REGIONS, pid=124}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/744bbec6934be304e01b009be760323d/.tmp/A/1b438dc27ed04c2c9f0e4ec68f60b1e0 is 50, key is test_row_0/A:col10/1732486560546/Put/seqid=0 2024-11-24T22:16:01,441 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45929 is added to blk_1073742290_1466 (size=12001) 2024-11-24T22:16:01,442 INFO [RS_FLUSH_OPERATIONS-regionserver/bd6592c37bb2:0-1 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=38.02 KB at sequenceid=81 (bloomFilter=true), to=hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/744bbec6934be304e01b009be760323d/.tmp/A/1b438dc27ed04c2c9f0e4ec68f60b1e0 2024-11-24T22:16:01,449 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bd6592c37bb2:0-1 {event_type=RS_FLUSH_REGIONS, pid=124}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/744bbec6934be304e01b009be760323d/.tmp/B/884de7f2215840a584587eb367958865 is 50, key is test_row_0/B:col10/1732486560546/Put/seqid=0 2024-11-24T22:16:01,454 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45929 is added to blk_1073742291_1467 (size=12001) 2024-11-24T22:16:01,457 INFO [RS_FLUSH_OPERATIONS-regionserver/bd6592c37bb2:0-1 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=38.02 KB at sequenceid=81 (bloomFilter=true), to=hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/744bbec6934be304e01b009be760323d/.tmp/B/884de7f2215840a584587eb367958865 2024-11-24T22:16:01,464 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bd6592c37bb2:0-1 {event_type=RS_FLUSH_REGIONS, pid=124}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/744bbec6934be304e01b009be760323d/.tmp/C/04d402c6aca248ec96f071d468ea7602 is 50, key is test_row_0/C:col10/1732486560546/Put/seqid=0 2024-11-24T22:16:01,467 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45929 is added to blk_1073742292_1468 (size=12001) 2024-11-24T22:16:01,747 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40087 {}] regionserver.HRegion(8855): Flush requested on 744bbec6934be304e01b009be760323d 2024-11-24T22:16:01,747 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732486556329.744bbec6934be304e01b009be760323d. as already flushing 2024-11-24T22:16:01,793 DEBUG [RS:0;bd6592c37bb2:40087-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/744bbec6934be304e01b009be760323d/.tmp/C/57a5000178934a4c98dbc71d7e5ffb12 as hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/744bbec6934be304e01b009be760323d/C/57a5000178934a4c98dbc71d7e5ffb12 2024-11-24T22:16:01,796 INFO [RS:0;bd6592c37bb2:40087-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 744bbec6934be304e01b009be760323d/C of 744bbec6934be304e01b009be760323d into 57a5000178934a4c98dbc71d7e5ffb12(size=11.8 K), total size for store is 11.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-24T22:16:01,796 DEBUG [RS:0;bd6592c37bb2:40087-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 744bbec6934be304e01b009be760323d: 2024-11-24T22:16:01,796 INFO [RS:0;bd6592c37bb2:40087-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732486556329.744bbec6934be304e01b009be760323d., storeName=744bbec6934be304e01b009be760323d/C, priority=13, startTime=1732486561311; duration=0sec 2024-11-24T22:16:01,796 DEBUG [RS:0;bd6592c37bb2:40087-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-24T22:16:01,796 DEBUG [RS:0;bd6592c37bb2:40087-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 744bbec6934be304e01b009be760323d:C 2024-11-24T22:16:01,801 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37701 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=123 2024-11-24T22:16:01,810 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40087 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=744bbec6934be304e01b009be760323d, server=bd6592c37bb2,40087,1732486423713 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-24T22:16:01,811 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40087 {}] ipc.CallRunner(138): callId: 53 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46792 deadline: 1732486621801, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=744bbec6934be304e01b009be760323d, server=bd6592c37bb2,40087,1732486423713 2024-11-24T22:16:01,811 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732486556329.744bbec6934be304e01b009be760323d., hostname=bd6592c37bb2,40087,1732486423713, seqNum=2 , the old value is region=TestAcidGuarantees,,1732486556329.744bbec6934be304e01b009be760323d., hostname=bd6592c37bb2,40087,1732486423713, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=744bbec6934be304e01b009be760323d, server=bd6592c37bb2,40087,1732486423713 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T22:16:01,811 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732486556329.744bbec6934be304e01b009be760323d., hostname=bd6592c37bb2,40087,1732486423713, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=744bbec6934be304e01b009be760323d, server=bd6592c37bb2,40087,1732486423713 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T22:16:01,811 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732486556329.744bbec6934be304e01b009be760323d., hostname=bd6592c37bb2,40087,1732486423713, seqNum=2 because the exception is null or not the one we care about 2024-11-24T22:16:01,868 INFO [RS_FLUSH_OPERATIONS-regionserver/bd6592c37bb2:0-1 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=38.02 KB at sequenceid=81 (bloomFilter=true), to=hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/744bbec6934be304e01b009be760323d/.tmp/C/04d402c6aca248ec96f071d468ea7602 2024-11-24T22:16:01,872 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bd6592c37bb2:0-1 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/744bbec6934be304e01b009be760323d/.tmp/A/1b438dc27ed04c2c9f0e4ec68f60b1e0 as hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/744bbec6934be304e01b009be760323d/A/1b438dc27ed04c2c9f0e4ec68f60b1e0 2024-11-24T22:16:01,875 INFO [RS_FLUSH_OPERATIONS-regionserver/bd6592c37bb2:0-1 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/744bbec6934be304e01b009be760323d/A/1b438dc27ed04c2c9f0e4ec68f60b1e0, entries=150, sequenceid=81, filesize=11.7 K 2024-11-24T22:16:01,875 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bd6592c37bb2:0-1 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/744bbec6934be304e01b009be760323d/.tmp/B/884de7f2215840a584587eb367958865 as hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/744bbec6934be304e01b009be760323d/B/884de7f2215840a584587eb367958865 2024-11-24T22:16:01,878 INFO [RS_FLUSH_OPERATIONS-regionserver/bd6592c37bb2:0-1 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/744bbec6934be304e01b009be760323d/B/884de7f2215840a584587eb367958865, entries=150, sequenceid=81, filesize=11.7 K 2024-11-24T22:16:01,879 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bd6592c37bb2:0-1 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/744bbec6934be304e01b009be760323d/.tmp/C/04d402c6aca248ec96f071d468ea7602 as hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/744bbec6934be304e01b009be760323d/C/04d402c6aca248ec96f071d468ea7602 2024-11-24T22:16:01,885 INFO [RS_FLUSH_OPERATIONS-regionserver/bd6592c37bb2:0-1 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/744bbec6934be304e01b009be760323d/C/04d402c6aca248ec96f071d468ea7602, entries=150, sequenceid=81, filesize=11.7 K 2024-11-24T22:16:01,885 INFO [RS_FLUSH_OPERATIONS-regionserver/bd6592c37bb2:0-1 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.HRegion(3140): Finished flush of dataSize ~114.05 KB/116790, heapSize ~339.38 KB/347520, currentSize=60.38 KB/61830 for 744bbec6934be304e01b009be760323d in 456ms, sequenceid=81, compaction requested=false 2024-11-24T22:16:01,886 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bd6592c37bb2:0-1 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.HRegion(2603): Flush status journal for 744bbec6934be304e01b009be760323d: 2024-11-24T22:16:01,886 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bd6592c37bb2:0-1 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732486556329.744bbec6934be304e01b009be760323d. 2024-11-24T22:16:01,886 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bd6592c37bb2:0-1 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=124 2024-11-24T22:16:01,886 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37701 {}] master.HMaster(4169): Remote procedure done, pid=124 2024-11-24T22:16:01,888 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=124, resume processing ppid=123 2024-11-24T22:16:01,888 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=124, ppid=123, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.2240 sec 2024-11-24T22:16:01,890 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=123, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=123, table=TestAcidGuarantees in 1.2280 sec 2024-11-24T22:16:01,925 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40087 {}] regionserver.HRegion(8855): Flush requested on 744bbec6934be304e01b009be760323d 2024-11-24T22:16:01,926 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 744bbec6934be304e01b009be760323d 3/3 column families, dataSize=67.09 KB heapSize=199.97 KB 2024-11-24T22:16:01,926 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 744bbec6934be304e01b009be760323d, store=A 2024-11-24T22:16:01,926 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-24T22:16:01,926 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 744bbec6934be304e01b009be760323d, store=B 2024-11-24T22:16:01,926 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-24T22:16:01,926 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 744bbec6934be304e01b009be760323d, store=C 2024-11-24T22:16:01,926 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-24T22:16:01,930 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/744bbec6934be304e01b009be760323d/.tmp/A/91027410d1ca4b5aa9256fa5d79892d5 is 50, key is test_row_0/A:col10/1732486561797/Put/seqid=0 2024-11-24T22:16:01,933 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45929 is added to blk_1073742293_1469 (size=14341) 2024-11-24T22:16:01,933 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=95 (bloomFilter=true), to=hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/744bbec6934be304e01b009be760323d/.tmp/A/91027410d1ca4b5aa9256fa5d79892d5 2024-11-24T22:16:01,941 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/744bbec6934be304e01b009be760323d/.tmp/B/2e3e28da1d0c43a1b0ce69ca0c9bebbb is 50, key is test_row_0/B:col10/1732486561797/Put/seqid=0 2024-11-24T22:16:01,946 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45929 is added to blk_1073742294_1470 (size=12001) 2024-11-24T22:16:02,086 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40087 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=744bbec6934be304e01b009be760323d, server=bd6592c37bb2,40087,1732486423713 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-24T22:16:02,086 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40087 {}] ipc.CallRunner(138): callId: 71 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46792 deadline: 1732486622075, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=744bbec6934be304e01b009be760323d, server=bd6592c37bb2,40087,1732486423713 2024-11-24T22:16:02,087 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732486556329.744bbec6934be304e01b009be760323d., hostname=bd6592c37bb2,40087,1732486423713, seqNum=2 , the old value is region=TestAcidGuarantees,,1732486556329.744bbec6934be304e01b009be760323d., hostname=bd6592c37bb2,40087,1732486423713, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=744bbec6934be304e01b009be760323d, server=bd6592c37bb2,40087,1732486423713 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T22:16:02,087 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732486556329.744bbec6934be304e01b009be760323d., hostname=bd6592c37bb2,40087,1732486423713, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=744bbec6934be304e01b009be760323d, server=bd6592c37bb2,40087,1732486423713 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T22:16:02,087 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732486556329.744bbec6934be304e01b009be760323d., hostname=bd6592c37bb2,40087,1732486423713, seqNum=2 because the exception is null or not the one we care about 2024-11-24T22:16:02,194 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40087 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=744bbec6934be304e01b009be760323d, server=bd6592c37bb2,40087,1732486423713 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-24T22:16:02,195 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40087 {}] ipc.CallRunner(138): callId: 72 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46792 deadline: 1732486622191, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=744bbec6934be304e01b009be760323d, server=bd6592c37bb2,40087,1732486423713 2024-11-24T22:16:02,195 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732486556329.744bbec6934be304e01b009be760323d., hostname=bd6592c37bb2,40087,1732486423713, seqNum=2 , the old value is region=TestAcidGuarantees,,1732486556329.744bbec6934be304e01b009be760323d., hostname=bd6592c37bb2,40087,1732486423713, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=744bbec6934be304e01b009be760323d, server=bd6592c37bb2,40087,1732486423713 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T22:16:02,195 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732486556329.744bbec6934be304e01b009be760323d., hostname=bd6592c37bb2,40087,1732486423713, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=744bbec6934be304e01b009be760323d, server=bd6592c37bb2,40087,1732486423713 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T22:16:02,195 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732486556329.744bbec6934be304e01b009be760323d., hostname=bd6592c37bb2,40087,1732486423713, seqNum=2 because the exception is null or not the one we care about 2024-11-24T22:16:02,347 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=95 (bloomFilter=true), to=hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/744bbec6934be304e01b009be760323d/.tmp/B/2e3e28da1d0c43a1b0ce69ca0c9bebbb 2024-11-24T22:16:02,353 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/744bbec6934be304e01b009be760323d/.tmp/C/a21678a1374048da9ca65db0afd717dc is 50, key is test_row_0/C:col10/1732486561797/Put/seqid=0 2024-11-24T22:16:02,357 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45929 is added to blk_1073742295_1471 (size=12001) 2024-11-24T22:16:02,409 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40087 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=744bbec6934be304e01b009be760323d, server=bd6592c37bb2,40087,1732486423713 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-24T22:16:02,409 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40087 {}] ipc.CallRunner(138): callId: 73 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46792 deadline: 1732486622400, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=744bbec6934be304e01b009be760323d, server=bd6592c37bb2,40087,1732486423713 2024-11-24T22:16:02,409 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732486556329.744bbec6934be304e01b009be760323d., hostname=bd6592c37bb2,40087,1732486423713, seqNum=2 , the old value is region=TestAcidGuarantees,,1732486556329.744bbec6934be304e01b009be760323d., hostname=bd6592c37bb2,40087,1732486423713, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=744bbec6934be304e01b009be760323d, server=bd6592c37bb2,40087,1732486423713 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T22:16:02,409 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732486556329.744bbec6934be304e01b009be760323d., hostname=bd6592c37bb2,40087,1732486423713, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=744bbec6934be304e01b009be760323d, server=bd6592c37bb2,40087,1732486423713 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T22:16:02,410 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732486556329.744bbec6934be304e01b009be760323d., hostname=bd6592c37bb2,40087,1732486423713, seqNum=2 because the exception is null or not the one we care about 2024-11-24T22:16:02,726 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40087 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=744bbec6934be304e01b009be760323d, server=bd6592c37bb2,40087,1732486423713 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-24T22:16:02,727 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40087 {}] ipc.CallRunner(138): callId: 74 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46792 deadline: 1732486622720, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=744bbec6934be304e01b009be760323d, server=bd6592c37bb2,40087,1732486423713 2024-11-24T22:16:02,727 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732486556329.744bbec6934be304e01b009be760323d., hostname=bd6592c37bb2,40087,1732486423713, seqNum=2 , the old value is region=TestAcidGuarantees,,1732486556329.744bbec6934be304e01b009be760323d., hostname=bd6592c37bb2,40087,1732486423713, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=744bbec6934be304e01b009be760323d, server=bd6592c37bb2,40087,1732486423713 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T22:16:02,727 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732486556329.744bbec6934be304e01b009be760323d., hostname=bd6592c37bb2,40087,1732486423713, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=744bbec6934be304e01b009be760323d, server=bd6592c37bb2,40087,1732486423713 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T22:16:02,727 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732486556329.744bbec6934be304e01b009be760323d., hostname=bd6592c37bb2,40087,1732486423713, seqNum=2 because the exception is null or not the one we care about 2024-11-24T22:16:02,758 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=95 (bloomFilter=true), to=hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/744bbec6934be304e01b009be760323d/.tmp/C/a21678a1374048da9ca65db0afd717dc 2024-11-24T22:16:02,762 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/744bbec6934be304e01b009be760323d/.tmp/A/91027410d1ca4b5aa9256fa5d79892d5 as hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/744bbec6934be304e01b009be760323d/A/91027410d1ca4b5aa9256fa5d79892d5 2024-11-24T22:16:02,765 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/744bbec6934be304e01b009be760323d/A/91027410d1ca4b5aa9256fa5d79892d5, entries=200, sequenceid=95, filesize=14.0 K 2024-11-24T22:16:02,765 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/744bbec6934be304e01b009be760323d/.tmp/B/2e3e28da1d0c43a1b0ce69ca0c9bebbb as hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/744bbec6934be304e01b009be760323d/B/2e3e28da1d0c43a1b0ce69ca0c9bebbb 2024-11-24T22:16:02,768 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/744bbec6934be304e01b009be760323d/B/2e3e28da1d0c43a1b0ce69ca0c9bebbb, entries=150, sequenceid=95, filesize=11.7 K 2024-11-24T22:16:02,769 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/744bbec6934be304e01b009be760323d/.tmp/C/a21678a1374048da9ca65db0afd717dc as hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/744bbec6934be304e01b009be760323d/C/a21678a1374048da9ca65db0afd717dc 2024-11-24T22:16:02,772 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/744bbec6934be304e01b009be760323d/C/a21678a1374048da9ca65db0afd717dc, entries=150, sequenceid=95, filesize=11.7 K 2024-11-24T22:16:02,773 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~67.09 KB/68700, heapSize ~199.92 KB/204720, currentSize=107.34 KB/109920 for 744bbec6934be304e01b009be760323d in 848ms, sequenceid=95, compaction requested=true 2024-11-24T22:16:02,773 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 744bbec6934be304e01b009be760323d: 2024-11-24T22:16:02,773 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 744bbec6934be304e01b009be760323d:A, priority=-2147483648, current under compaction store size is 1 2024-11-24T22:16:02,773 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-24T22:16:02,773 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 744bbec6934be304e01b009be760323d:B, priority=-2147483648, current under compaction store size is 2 2024-11-24T22:16:02,773 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-24T22:16:02,773 DEBUG [RS:0;bd6592c37bb2:40087-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-24T22:16:02,773 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 744bbec6934be304e01b009be760323d:C, priority=-2147483648, current under compaction store size is 3 2024-11-24T22:16:02,773 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-24T22:16:02,773 DEBUG [RS:0;bd6592c37bb2:40087-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-24T22:16:02,774 DEBUG [RS:0;bd6592c37bb2:40087-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36106 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-24T22:16:02,774 DEBUG [RS:0;bd6592c37bb2:40087-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 38446 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-24T22:16:02,774 DEBUG [RS:0;bd6592c37bb2:40087-shortCompactions-0 {}] regionserver.HStore(1541): 744bbec6934be304e01b009be760323d/A is initiating minor compaction (all files) 2024-11-24T22:16:02,774 DEBUG [RS:0;bd6592c37bb2:40087-longCompactions-0 {}] regionserver.HStore(1541): 744bbec6934be304e01b009be760323d/B is initiating minor compaction (all files) 2024-11-24T22:16:02,774 INFO [RS:0;bd6592c37bb2:40087-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 744bbec6934be304e01b009be760323d/B in TestAcidGuarantees,,1732486556329.744bbec6934be304e01b009be760323d. 2024-11-24T22:16:02,774 INFO [RS:0;bd6592c37bb2:40087-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 744bbec6934be304e01b009be760323d/A in TestAcidGuarantees,,1732486556329.744bbec6934be304e01b009be760323d. 2024-11-24T22:16:02,774 INFO [RS:0;bd6592c37bb2:40087-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/744bbec6934be304e01b009be760323d/B/1a3fc19c62624ad0a67ac686e829085c, hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/744bbec6934be304e01b009be760323d/B/884de7f2215840a584587eb367958865, hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/744bbec6934be304e01b009be760323d/B/2e3e28da1d0c43a1b0ce69ca0c9bebbb] into tmpdir=hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/744bbec6934be304e01b009be760323d/.tmp, totalSize=35.3 K 2024-11-24T22:16:02,774 INFO [RS:0;bd6592c37bb2:40087-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/744bbec6934be304e01b009be760323d/A/6a62440767db4e10be88de20dd521d76, hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/744bbec6934be304e01b009be760323d/A/1b438dc27ed04c2c9f0e4ec68f60b1e0, hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/744bbec6934be304e01b009be760323d/A/91027410d1ca4b5aa9256fa5d79892d5] into tmpdir=hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/744bbec6934be304e01b009be760323d/.tmp, totalSize=37.5 K 2024-11-24T22:16:02,775 DEBUG [RS:0;bd6592c37bb2:40087-longCompactions-0 {}] compactions.Compactor(225): Compacting 1a3fc19c62624ad0a67ac686e829085c, keycount=150, bloomtype=ROW, size=11.8 K, encoding=NONE, compression=NONE, seqNum=59, earliestPutTs=1732486559779 2024-11-24T22:16:02,775 DEBUG [RS:0;bd6592c37bb2:40087-shortCompactions-0 {}] compactions.Compactor(225): Compacting 6a62440767db4e10be88de20dd521d76, keycount=150, bloomtype=ROW, size=11.8 K, encoding=NONE, compression=NONE, seqNum=59, earliestPutTs=1732486559779 2024-11-24T22:16:02,775 DEBUG [RS:0;bd6592c37bb2:40087-longCompactions-0 {}] compactions.Compactor(225): Compacting 884de7f2215840a584587eb367958865, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=81, earliestPutTs=1732486560527 2024-11-24T22:16:02,775 DEBUG [RS:0;bd6592c37bb2:40087-shortCompactions-0 {}] compactions.Compactor(225): Compacting 1b438dc27ed04c2c9f0e4ec68f60b1e0, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=81, earliestPutTs=1732486560527 2024-11-24T22:16:02,775 DEBUG [RS:0;bd6592c37bb2:40087-longCompactions-0 {}] compactions.Compactor(225): Compacting 2e3e28da1d0c43a1b0ce69ca0c9bebbb, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=95, earliestPutTs=1732486561797 2024-11-24T22:16:02,775 DEBUG [RS:0;bd6592c37bb2:40087-shortCompactions-0 {}] compactions.Compactor(225): Compacting 91027410d1ca4b5aa9256fa5d79892d5, keycount=200, bloomtype=ROW, size=14.0 K, encoding=NONE, compression=NONE, seqNum=95, earliestPutTs=1732486561782 2024-11-24T22:16:02,782 INFO [RS:0;bd6592c37bb2:40087-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 744bbec6934be304e01b009be760323d#A#compaction#392 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 57.69 MB/second 2024-11-24T22:16:02,782 INFO [RS:0;bd6592c37bb2:40087-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 744bbec6934be304e01b009be760323d#B#compaction#393 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 57.69 MB/second 2024-11-24T22:16:02,782 DEBUG [RS:0;bd6592c37bb2:40087-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/744bbec6934be304e01b009be760323d/.tmp/A/85a7f1a085d7453a819dea01d918cb9a is 50, key is test_row_0/A:col10/1732486561797/Put/seqid=0 2024-11-24T22:16:02,783 DEBUG [RS:0;bd6592c37bb2:40087-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/744bbec6934be304e01b009be760323d/.tmp/B/9b45215aecb6437aa1a20c2a2e6823e5 is 50, key is test_row_0/B:col10/1732486561797/Put/seqid=0 2024-11-24T22:16:02,787 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45929 is added to blk_1073742296_1472 (size=12207) 2024-11-24T22:16:02,788 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45929 is added to blk_1073742297_1473 (size=12207) 2024-11-24T22:16:02,811 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37701 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=123 2024-11-24T22:16:02,811 INFO [RPCClient-NioEventLoopGroup-4-10 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-24T22:16:02,812 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37701 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-24T22:16:02,813 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37701 {}] procedure2.ProcedureExecutor(1139): Stored pid=125, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=125, table=TestAcidGuarantees 2024-11-24T22:16:02,813 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37701 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=125 2024-11-24T22:16:02,813 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=125, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=125, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-24T22:16:02,814 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=125, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=125, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-24T22:16:02,814 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=126, ppid=125, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-24T22:16:02,873 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40087 {}] regionserver.HRegion(8855): Flush requested on 744bbec6934be304e01b009be760323d 2024-11-24T22:16:02,873 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 744bbec6934be304e01b009be760323d 3/3 column families, dataSize=114.05 KB heapSize=339.42 KB 2024-11-24T22:16:02,874 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 744bbec6934be304e01b009be760323d, store=A 2024-11-24T22:16:02,874 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-24T22:16:02,874 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 744bbec6934be304e01b009be760323d, store=B 2024-11-24T22:16:02,874 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-24T22:16:02,874 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 744bbec6934be304e01b009be760323d, store=C 2024-11-24T22:16:02,874 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-24T22:16:02,879 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/744bbec6934be304e01b009be760323d/.tmp/A/b78790bd2d7849b18fe9b44f38b57ece is 50, key is test_row_0/A:col10/1732486562047/Put/seqid=0 2024-11-24T22:16:02,882 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45929 is added to blk_1073742298_1474 (size=14341) 2024-11-24T22:16:02,920 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37701 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=125 2024-11-24T22:16:02,933 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40087 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=744bbec6934be304e01b009be760323d, server=bd6592c37bb2,40087,1732486423713 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-24T22:16:02,933 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40087 {}] ipc.CallRunner(138): callId: 17 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46842 deadline: 1732486622916, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=744bbec6934be304e01b009be760323d, server=bd6592c37bb2,40087,1732486423713 2024-11-24T22:16:02,933 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40087 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=744bbec6934be304e01b009be760323d, server=bd6592c37bb2,40087,1732486423713 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-24T22:16:02,933 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40087 {}] ipc.CallRunner(138): callId: 17 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46816 deadline: 1732486622928, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=744bbec6934be304e01b009be760323d, server=bd6592c37bb2,40087,1732486423713 2024-11-24T22:16:02,933 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732486556329.744bbec6934be304e01b009be760323d., hostname=bd6592c37bb2,40087,1732486423713, seqNum=2 , the old value is region=TestAcidGuarantees,,1732486556329.744bbec6934be304e01b009be760323d., hostname=bd6592c37bb2,40087,1732486423713, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=744bbec6934be304e01b009be760323d, server=bd6592c37bb2,40087,1732486423713 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T22:16:02,933 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732486556329.744bbec6934be304e01b009be760323d., hostname=bd6592c37bb2,40087,1732486423713, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=744bbec6934be304e01b009be760323d, server=bd6592c37bb2,40087,1732486423713 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T22:16:02,934 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732486556329.744bbec6934be304e01b009be760323d., hostname=bd6592c37bb2,40087,1732486423713, seqNum=2 because the exception is null or not the one we care about 2024-11-24T22:16:02,934 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732486556329.744bbec6934be304e01b009be760323d., hostname=bd6592c37bb2,40087,1732486423713, seqNum=2 , the old value is region=TestAcidGuarantees,,1732486556329.744bbec6934be304e01b009be760323d., hostname=bd6592c37bb2,40087,1732486423713, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=744bbec6934be304e01b009be760323d, server=bd6592c37bb2,40087,1732486423713 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T22:16:02,934 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732486556329.744bbec6934be304e01b009be760323d., hostname=bd6592c37bb2,40087,1732486423713, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=744bbec6934be304e01b009be760323d, server=bd6592c37bb2,40087,1732486423713 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T22:16:02,934 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732486556329.744bbec6934be304e01b009be760323d., hostname=bd6592c37bb2,40087,1732486423713, seqNum=2 because the exception is null or not the one we care about 2024-11-24T22:16:02,938 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40087 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=744bbec6934be304e01b009be760323d, server=bd6592c37bb2,40087,1732486423713 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-24T22:16:02,938 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40087 {}] ipc.CallRunner(138): callId: 20 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46824 deadline: 1732486622933, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=744bbec6934be304e01b009be760323d, server=bd6592c37bb2,40087,1732486423713 2024-11-24T22:16:02,939 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732486556329.744bbec6934be304e01b009be760323d., hostname=bd6592c37bb2,40087,1732486423713, seqNum=2 , the old value is region=TestAcidGuarantees,,1732486556329.744bbec6934be304e01b009be760323d., hostname=bd6592c37bb2,40087,1732486423713, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=744bbec6934be304e01b009be760323d, server=bd6592c37bb2,40087,1732486423713 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T22:16:02,939 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40087 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=744bbec6934be304e01b009be760323d, server=bd6592c37bb2,40087,1732486423713 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-24T22:16:02,939 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732486556329.744bbec6934be304e01b009be760323d., hostname=bd6592c37bb2,40087,1732486423713, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=744bbec6934be304e01b009be760323d, server=bd6592c37bb2,40087,1732486423713 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T22:16:02,939 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732486556329.744bbec6934be304e01b009be760323d., hostname=bd6592c37bb2,40087,1732486423713, seqNum=2 because the exception is null or not the one we care about 2024-11-24T22:16:02,939 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40087 {}] ipc.CallRunner(138): callId: 18 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46820 deadline: 1732486622933, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=744bbec6934be304e01b009be760323d, server=bd6592c37bb2,40087,1732486423713 2024-11-24T22:16:02,939 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732486556329.744bbec6934be304e01b009be760323d., hostname=bd6592c37bb2,40087,1732486423713, seqNum=2 , the old value is region=TestAcidGuarantees,,1732486556329.744bbec6934be304e01b009be760323d., hostname=bd6592c37bb2,40087,1732486423713, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=744bbec6934be304e01b009be760323d, server=bd6592c37bb2,40087,1732486423713 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T22:16:02,939 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732486556329.744bbec6934be304e01b009be760323d., hostname=bd6592c37bb2,40087,1732486423713, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=744bbec6934be304e01b009be760323d, server=bd6592c37bb2,40087,1732486423713 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T22:16:02,939 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732486556329.744bbec6934be304e01b009be760323d., hostname=bd6592c37bb2,40087,1732486423713, seqNum=2 because the exception is null or not the one we care about 2024-11-24T22:16:02,966 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=40087 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=126 2024-11-24T22:16:02,966 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bd6592c37bb2:0-2 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732486556329.744bbec6934be304e01b009be760323d. 2024-11-24T22:16:02,966 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bd6592c37bb2:0-2 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732486556329.744bbec6934be304e01b009be760323d. as already flushing 2024-11-24T22:16:02,966 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bd6592c37bb2:0-2 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732486556329.744bbec6934be304e01b009be760323d. 2024-11-24T22:16:02,966 ERROR [RS_FLUSH_OPERATIONS-regionserver/bd6592c37bb2:0-2 {event_type=RS_FLUSH_REGIONS, pid=126}] handler.RSProcedureHandler(58): pid=126 java.io.IOException: Unable to complete flush {ENCODED => 744bbec6934be304e01b009be760323d, NAME => 'TestAcidGuarantees,,1732486556329.744bbec6934be304e01b009be760323d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T22:16:02,967 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bd6592c37bb2:0-2 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=126 java.io.IOException: Unable to complete flush {ENCODED => 744bbec6934be304e01b009be760323d, NAME => 'TestAcidGuarantees,,1732486556329.744bbec6934be304e01b009be760323d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T22:16:02,967 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37701 {}] master.HMaster(4177): Remote procedure failed, pid=126 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 744bbec6934be304e01b009be760323d, NAME => 'TestAcidGuarantees,,1732486556329.744bbec6934be304e01b009be760323d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 744bbec6934be304e01b009be760323d, NAME => 'TestAcidGuarantees,,1732486556329.744bbec6934be304e01b009be760323d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T22:16:03,053 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40087 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=744bbec6934be304e01b009be760323d, server=bd6592c37bb2,40087,1732486423713 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-24T22:16:03,053 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40087 {}] ipc.CallRunner(138): callId: 18 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46842 deadline: 1732486623040, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=744bbec6934be304e01b009be760323d, server=bd6592c37bb2,40087,1732486423713 2024-11-24T22:16:03,054 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732486556329.744bbec6934be304e01b009be760323d., hostname=bd6592c37bb2,40087,1732486423713, seqNum=2 , the old value is region=TestAcidGuarantees,,1732486556329.744bbec6934be304e01b009be760323d., hostname=bd6592c37bb2,40087,1732486423713, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=744bbec6934be304e01b009be760323d, server=bd6592c37bb2,40087,1732486423713 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T22:16:03,053 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40087 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=744bbec6934be304e01b009be760323d, server=bd6592c37bb2,40087,1732486423713 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-24T22:16:03,054 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732486556329.744bbec6934be304e01b009be760323d., hostname=bd6592c37bb2,40087,1732486423713, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=744bbec6934be304e01b009be760323d, server=bd6592c37bb2,40087,1732486423713 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T22:16:03,054 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732486556329.744bbec6934be304e01b009be760323d., hostname=bd6592c37bb2,40087,1732486423713, seqNum=2 because the exception is null or not the one we care about 2024-11-24T22:16:03,054 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40087 {}] ipc.CallRunner(138): callId: 18 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46816 deadline: 1732486623040, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=744bbec6934be304e01b009be760323d, server=bd6592c37bb2,40087,1732486423713 2024-11-24T22:16:03,054 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732486556329.744bbec6934be304e01b009be760323d., hostname=bd6592c37bb2,40087,1732486423713, seqNum=2 , the old value is region=TestAcidGuarantees,,1732486556329.744bbec6934be304e01b009be760323d., hostname=bd6592c37bb2,40087,1732486423713, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=744bbec6934be304e01b009be760323d, server=bd6592c37bb2,40087,1732486423713 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T22:16:03,054 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732486556329.744bbec6934be304e01b009be760323d., hostname=bd6592c37bb2,40087,1732486423713, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=744bbec6934be304e01b009be760323d, server=bd6592c37bb2,40087,1732486423713 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T22:16:03,054 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732486556329.744bbec6934be304e01b009be760323d., hostname=bd6592c37bb2,40087,1732486423713, seqNum=2 because the exception is null or not the one we care about 2024-11-24T22:16:03,054 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40087 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=744bbec6934be304e01b009be760323d, server=bd6592c37bb2,40087,1732486423713 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-24T22:16:03,054 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40087 {}] ipc.CallRunner(138): callId: 21 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46824 deadline: 1732486623040, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=744bbec6934be304e01b009be760323d, server=bd6592c37bb2,40087,1732486423713 2024-11-24T22:16:03,054 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732486556329.744bbec6934be304e01b009be760323d., hostname=bd6592c37bb2,40087,1732486423713, seqNum=2 , the old value is region=TestAcidGuarantees,,1732486556329.744bbec6934be304e01b009be760323d., hostname=bd6592c37bb2,40087,1732486423713, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=744bbec6934be304e01b009be760323d, server=bd6592c37bb2,40087,1732486423713 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T22:16:03,054 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732486556329.744bbec6934be304e01b009be760323d., hostname=bd6592c37bb2,40087,1732486423713, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=744bbec6934be304e01b009be760323d, server=bd6592c37bb2,40087,1732486423713 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T22:16:03,054 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732486556329.744bbec6934be304e01b009be760323d., hostname=bd6592c37bb2,40087,1732486423713, seqNum=2 because the exception is null or not the one we care about 2024-11-24T22:16:03,054 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40087 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=744bbec6934be304e01b009be760323d, server=bd6592c37bb2,40087,1732486423713 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-24T22:16:03,054 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40087 {}] ipc.CallRunner(138): callId: 19 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46820 deadline: 1732486623041, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=744bbec6934be304e01b009be760323d, server=bd6592c37bb2,40087,1732486423713 2024-11-24T22:16:03,055 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732486556329.744bbec6934be304e01b009be760323d., hostname=bd6592c37bb2,40087,1732486423713, seqNum=2 , the old value is region=TestAcidGuarantees,,1732486556329.744bbec6934be304e01b009be760323d., hostname=bd6592c37bb2,40087,1732486423713, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=744bbec6934be304e01b009be760323d, server=bd6592c37bb2,40087,1732486423713 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T22:16:03,055 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732486556329.744bbec6934be304e01b009be760323d., hostname=bd6592c37bb2,40087,1732486423713, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=744bbec6934be304e01b009be760323d, server=bd6592c37bb2,40087,1732486423713 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T22:16:03,055 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732486556329.744bbec6934be304e01b009be760323d., hostname=bd6592c37bb2,40087,1732486423713, seqNum=2 because the exception is null or not the one we care about 2024-11-24T22:16:03,119 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=40087 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=126 2024-11-24T22:16:03,119 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bd6592c37bb2:0-0 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732486556329.744bbec6934be304e01b009be760323d. 2024-11-24T22:16:03,119 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bd6592c37bb2:0-0 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732486556329.744bbec6934be304e01b009be760323d. as already flushing 2024-11-24T22:16:03,120 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bd6592c37bb2:0-0 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732486556329.744bbec6934be304e01b009be760323d. 2024-11-24T22:16:03,120 ERROR [RS_FLUSH_OPERATIONS-regionserver/bd6592c37bb2:0-0 {event_type=RS_FLUSH_REGIONS, pid=126}] handler.RSProcedureHandler(58): pid=126 java.io.IOException: Unable to complete flush {ENCODED => 744bbec6934be304e01b009be760323d, NAME => 'TestAcidGuarantees,,1732486556329.744bbec6934be304e01b009be760323d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T22:16:03,120 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bd6592c37bb2:0-0 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=126 java.io.IOException: Unable to complete flush {ENCODED => 744bbec6934be304e01b009be760323d, NAME => 'TestAcidGuarantees,,1732486556329.744bbec6934be304e01b009be760323d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T22:16:03,120 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37701 {}] master.HMaster(4177): Remote procedure failed, pid=126 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 744bbec6934be304e01b009be760323d, NAME => 'TestAcidGuarantees,,1732486556329.744bbec6934be304e01b009be760323d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 744bbec6934be304e01b009be760323d, NAME => 'TestAcidGuarantees,,1732486556329.744bbec6934be304e01b009be760323d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T22:16:03,130 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37701 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=125 2024-11-24T22:16:03,190 DEBUG [RS:0;bd6592c37bb2:40087-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/744bbec6934be304e01b009be760323d/.tmp/B/9b45215aecb6437aa1a20c2a2e6823e5 as hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/744bbec6934be304e01b009be760323d/B/9b45215aecb6437aa1a20c2a2e6823e5 2024-11-24T22:16:03,190 DEBUG [RS:0;bd6592c37bb2:40087-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/744bbec6934be304e01b009be760323d/.tmp/A/85a7f1a085d7453a819dea01d918cb9a as hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/744bbec6934be304e01b009be760323d/A/85a7f1a085d7453a819dea01d918cb9a 2024-11-24T22:16:03,194 INFO [RS:0;bd6592c37bb2:40087-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 744bbec6934be304e01b009be760323d/B of 744bbec6934be304e01b009be760323d into 9b45215aecb6437aa1a20c2a2e6823e5(size=11.9 K), total size for store is 11.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-24T22:16:03,194 INFO [RS:0;bd6592c37bb2:40087-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 744bbec6934be304e01b009be760323d/A of 744bbec6934be304e01b009be760323d into 85a7f1a085d7453a819dea01d918cb9a(size=11.9 K), total size for store is 11.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-24T22:16:03,194 DEBUG [RS:0;bd6592c37bb2:40087-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 744bbec6934be304e01b009be760323d: 2024-11-24T22:16:03,194 DEBUG [RS:0;bd6592c37bb2:40087-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 744bbec6934be304e01b009be760323d: 2024-11-24T22:16:03,194 INFO [RS:0;bd6592c37bb2:40087-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732486556329.744bbec6934be304e01b009be760323d., storeName=744bbec6934be304e01b009be760323d/B, priority=13, startTime=1732486562773; duration=0sec 2024-11-24T22:16:03,194 INFO [RS:0;bd6592c37bb2:40087-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732486556329.744bbec6934be304e01b009be760323d., storeName=744bbec6934be304e01b009be760323d/A, priority=13, startTime=1732486562773; duration=0sec 2024-11-24T22:16:03,194 DEBUG [RS:0;bd6592c37bb2:40087-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-24T22:16:03,195 DEBUG [RS:0;bd6592c37bb2:40087-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 744bbec6934be304e01b009be760323d:B 2024-11-24T22:16:03,195 DEBUG [RS:0;bd6592c37bb2:40087-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-24T22:16:03,195 DEBUG [RS:0;bd6592c37bb2:40087-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 744bbec6934be304e01b009be760323d:A 2024-11-24T22:16:03,195 DEBUG [RS:0;bd6592c37bb2:40087-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-24T22:16:03,195 DEBUG [RS:0;bd6592c37bb2:40087-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36106 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-24T22:16:03,195 DEBUG [RS:0;bd6592c37bb2:40087-longCompactions-0 {}] regionserver.HStore(1541): 744bbec6934be304e01b009be760323d/C is initiating minor compaction (all files) 2024-11-24T22:16:03,195 INFO [RS:0;bd6592c37bb2:40087-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 744bbec6934be304e01b009be760323d/C in TestAcidGuarantees,,1732486556329.744bbec6934be304e01b009be760323d. 2024-11-24T22:16:03,195 INFO [RS:0;bd6592c37bb2:40087-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/744bbec6934be304e01b009be760323d/C/57a5000178934a4c98dbc71d7e5ffb12, hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/744bbec6934be304e01b009be760323d/C/04d402c6aca248ec96f071d468ea7602, hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/744bbec6934be304e01b009be760323d/C/a21678a1374048da9ca65db0afd717dc] into tmpdir=hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/744bbec6934be304e01b009be760323d/.tmp, totalSize=35.3 K 2024-11-24T22:16:03,196 DEBUG [RS:0;bd6592c37bb2:40087-longCompactions-0 {}] compactions.Compactor(225): Compacting 57a5000178934a4c98dbc71d7e5ffb12, keycount=150, bloomtype=ROW, size=11.8 K, encoding=NONE, compression=NONE, seqNum=59, earliestPutTs=1732486559779 2024-11-24T22:16:03,196 DEBUG [RS:0;bd6592c37bb2:40087-longCompactions-0 {}] compactions.Compactor(225): Compacting 04d402c6aca248ec96f071d468ea7602, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=81, earliestPutTs=1732486560527 2024-11-24T22:16:03,196 DEBUG [RS:0;bd6592c37bb2:40087-longCompactions-0 {}] compactions.Compactor(225): Compacting a21678a1374048da9ca65db0afd717dc, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=95, earliestPutTs=1732486561797 2024-11-24T22:16:03,205 INFO [RS:0;bd6592c37bb2:40087-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 744bbec6934be304e01b009be760323d#C#compaction#395 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 57.69 MB/second 2024-11-24T22:16:03,206 DEBUG [RS:0;bd6592c37bb2:40087-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/744bbec6934be304e01b009be760323d/.tmp/C/dfd78c5f1e3446ceb9e6877167ddb6c0 is 50, key is test_row_0/C:col10/1732486561797/Put/seqid=0 2024-11-24T22:16:03,209 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45929 is added to blk_1073742299_1475 (size=12207) 2024-11-24T22:16:03,242 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40087 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=744bbec6934be304e01b009be760323d, server=bd6592c37bb2,40087,1732486423713 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-24T22:16:03,242 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40087 {}] ipc.CallRunner(138): callId: 75 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46792 deadline: 1732486623230, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=744bbec6934be304e01b009be760323d, server=bd6592c37bb2,40087,1732486423713 2024-11-24T22:16:03,242 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732486556329.744bbec6934be304e01b009be760323d., hostname=bd6592c37bb2,40087,1732486423713, seqNum=2 , the old value is region=TestAcidGuarantees,,1732486556329.744bbec6934be304e01b009be760323d., hostname=bd6592c37bb2,40087,1732486423713, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=744bbec6934be304e01b009be760323d, server=bd6592c37bb2,40087,1732486423713 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T22:16:03,242 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732486556329.744bbec6934be304e01b009be760323d., hostname=bd6592c37bb2,40087,1732486423713, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=744bbec6934be304e01b009be760323d, server=bd6592c37bb2,40087,1732486423713 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T22:16:03,242 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732486556329.744bbec6934be304e01b009be760323d., hostname=bd6592c37bb2,40087,1732486423713, seqNum=2 because the exception is null or not the one we care about 2024-11-24T22:16:03,271 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40087 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=744bbec6934be304e01b009be760323d, server=bd6592c37bb2,40087,1732486423713 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-24T22:16:03,271 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40087 {}] ipc.CallRunner(138): callId: 19 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46842 deadline: 1732486623261, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=744bbec6934be304e01b009be760323d, server=bd6592c37bb2,40087,1732486423713 2024-11-24T22:16:03,271 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40087 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=744bbec6934be304e01b009be760323d, server=bd6592c37bb2,40087,1732486423713 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-24T22:16:03,271 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732486556329.744bbec6934be304e01b009be760323d., hostname=bd6592c37bb2,40087,1732486423713, seqNum=2 , the old value is region=TestAcidGuarantees,,1732486556329.744bbec6934be304e01b009be760323d., hostname=bd6592c37bb2,40087,1732486423713, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=744bbec6934be304e01b009be760323d, server=bd6592c37bb2,40087,1732486423713 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T22:16:03,271 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40087 {}] ipc.CallRunner(138): callId: 19 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46816 deadline: 1732486623261, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=744bbec6934be304e01b009be760323d, server=bd6592c37bb2,40087,1732486423713 2024-11-24T22:16:03,271 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732486556329.744bbec6934be304e01b009be760323d., hostname=bd6592c37bb2,40087,1732486423713, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=744bbec6934be304e01b009be760323d, server=bd6592c37bb2,40087,1732486423713 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T22:16:03,272 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732486556329.744bbec6934be304e01b009be760323d., hostname=bd6592c37bb2,40087,1732486423713, seqNum=2 because the exception is null or not the one we care about 2024-11-24T22:16:03,272 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40087 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=744bbec6934be304e01b009be760323d, server=bd6592c37bb2,40087,1732486423713 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-24T22:16:03,272 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732486556329.744bbec6934be304e01b009be760323d., hostname=bd6592c37bb2,40087,1732486423713, seqNum=2 , the old value is region=TestAcidGuarantees,,1732486556329.744bbec6934be304e01b009be760323d., hostname=bd6592c37bb2,40087,1732486423713, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=744bbec6934be304e01b009be760323d, server=bd6592c37bb2,40087,1732486423713 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T22:16:03,272 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732486556329.744bbec6934be304e01b009be760323d., hostname=bd6592c37bb2,40087,1732486423713, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=744bbec6934be304e01b009be760323d, server=bd6592c37bb2,40087,1732486423713 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T22:16:03,272 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40087 {}] ipc.CallRunner(138): callId: 22 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46824 deadline: 1732486623261, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=744bbec6934be304e01b009be760323d, server=bd6592c37bb2,40087,1732486423713 2024-11-24T22:16:03,272 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732486556329.744bbec6934be304e01b009be760323d., hostname=bd6592c37bb2,40087,1732486423713, seqNum=2 because the exception is null or not the one we care about 2024-11-24T22:16:03,272 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40087 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=744bbec6934be304e01b009be760323d, server=bd6592c37bb2,40087,1732486423713 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-24T22:16:03,272 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732486556329.744bbec6934be304e01b009be760323d., hostname=bd6592c37bb2,40087,1732486423713, seqNum=2 , the old value is region=TestAcidGuarantees,,1732486556329.744bbec6934be304e01b009be760323d., hostname=bd6592c37bb2,40087,1732486423713, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=744bbec6934be304e01b009be760323d, server=bd6592c37bb2,40087,1732486423713 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T22:16:03,272 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732486556329.744bbec6934be304e01b009be760323d., hostname=bd6592c37bb2,40087,1732486423713, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=744bbec6934be304e01b009be760323d, server=bd6592c37bb2,40087,1732486423713 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T22:16:03,272 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40087 {}] ipc.CallRunner(138): callId: 20 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46820 deadline: 1732486623261, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=744bbec6934be304e01b009be760323d, server=bd6592c37bb2,40087,1732486423713 2024-11-24T22:16:03,272 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732486556329.744bbec6934be304e01b009be760323d., hostname=bd6592c37bb2,40087,1732486423713, seqNum=2 because the exception is null or not the one we care about 2024-11-24T22:16:03,272 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=40087 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=126 2024-11-24T22:16:03,272 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732486556329.744bbec6934be304e01b009be760323d., hostname=bd6592c37bb2,40087,1732486423713, seqNum=2 , the old value is region=TestAcidGuarantees,,1732486556329.744bbec6934be304e01b009be760323d., hostname=bd6592c37bb2,40087,1732486423713, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=744bbec6934be304e01b009be760323d, server=bd6592c37bb2,40087,1732486423713 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T22:16:03,272 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732486556329.744bbec6934be304e01b009be760323d., hostname=bd6592c37bb2,40087,1732486423713, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=744bbec6934be304e01b009be760323d, server=bd6592c37bb2,40087,1732486423713 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T22:16:03,273 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732486556329.744bbec6934be304e01b009be760323d., hostname=bd6592c37bb2,40087,1732486423713, seqNum=2 because the exception is null or not the one we care about 2024-11-24T22:16:03,273 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bd6592c37bb2:0-1 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732486556329.744bbec6934be304e01b009be760323d. 2024-11-24T22:16:03,273 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bd6592c37bb2:0-1 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732486556329.744bbec6934be304e01b009be760323d. as already flushing 2024-11-24T22:16:03,273 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bd6592c37bb2:0-1 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732486556329.744bbec6934be304e01b009be760323d. 2024-11-24T22:16:03,273 ERROR [RS_FLUSH_OPERATIONS-regionserver/bd6592c37bb2:0-1 {event_type=RS_FLUSH_REGIONS, pid=126}] handler.RSProcedureHandler(58): pid=126 java.io.IOException: Unable to complete flush {ENCODED => 744bbec6934be304e01b009be760323d, NAME => 'TestAcidGuarantees,,1732486556329.744bbec6934be304e01b009be760323d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T22:16:03,273 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bd6592c37bb2:0-1 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=126 java.io.IOException: Unable to complete flush {ENCODED => 744bbec6934be304e01b009be760323d, NAME => 'TestAcidGuarantees,,1732486556329.744bbec6934be304e01b009be760323d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T22:16:03,274 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37701 {}] master.HMaster(4177): Remote procedure failed, pid=126 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 744bbec6934be304e01b009be760323d, NAME => 'TestAcidGuarantees,,1732486556329.744bbec6934be304e01b009be760323d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 744bbec6934be304e01b009be760323d, NAME => 'TestAcidGuarantees,,1732486556329.744bbec6934be304e01b009be760323d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T22:16:03,283 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=38.02 KB at sequenceid=115 (bloomFilter=true), to=hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/744bbec6934be304e01b009be760323d/.tmp/A/b78790bd2d7849b18fe9b44f38b57ece 2024-11-24T22:16:03,290 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/744bbec6934be304e01b009be760323d/.tmp/B/04e097aec304409694fd69e1932b2a7b is 50, key is test_row_0/B:col10/1732486562047/Put/seqid=0 2024-11-24T22:16:03,294 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45929 is added to blk_1073742300_1476 (size=12001) 2024-11-24T22:16:03,425 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=40087 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=126 2024-11-24T22:16:03,426 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bd6592c37bb2:0-2 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732486556329.744bbec6934be304e01b009be760323d. 2024-11-24T22:16:03,426 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bd6592c37bb2:0-2 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732486556329.744bbec6934be304e01b009be760323d. as already flushing 2024-11-24T22:16:03,426 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bd6592c37bb2:0-2 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732486556329.744bbec6934be304e01b009be760323d. 2024-11-24T22:16:03,426 ERROR [RS_FLUSH_OPERATIONS-regionserver/bd6592c37bb2:0-2 {event_type=RS_FLUSH_REGIONS, pid=126}] handler.RSProcedureHandler(58): pid=126 java.io.IOException: Unable to complete flush {ENCODED => 744bbec6934be304e01b009be760323d, NAME => 'TestAcidGuarantees,,1732486556329.744bbec6934be304e01b009be760323d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T22:16:03,426 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bd6592c37bb2:0-2 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=126 java.io.IOException: Unable to complete flush {ENCODED => 744bbec6934be304e01b009be760323d, NAME => 'TestAcidGuarantees,,1732486556329.744bbec6934be304e01b009be760323d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T22:16:03,426 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37701 {}] master.HMaster(4177): Remote procedure failed, pid=126 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 744bbec6934be304e01b009be760323d, NAME => 'TestAcidGuarantees,,1732486556329.744bbec6934be304e01b009be760323d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 744bbec6934be304e01b009be760323d, NAME => 'TestAcidGuarantees,,1732486556329.744bbec6934be304e01b009be760323d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T22:16:03,440 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37701 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=125 2024-11-24T22:16:03,578 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=40087 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=126 2024-11-24T22:16:03,578 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bd6592c37bb2:0-0 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732486556329.744bbec6934be304e01b009be760323d. 2024-11-24T22:16:03,578 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bd6592c37bb2:0-0 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732486556329.744bbec6934be304e01b009be760323d. as already flushing 2024-11-24T22:16:03,578 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bd6592c37bb2:0-0 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732486556329.744bbec6934be304e01b009be760323d. 2024-11-24T22:16:03,579 ERROR [RS_FLUSH_OPERATIONS-regionserver/bd6592c37bb2:0-0 {event_type=RS_FLUSH_REGIONS, pid=126}] handler.RSProcedureHandler(58): pid=126 java.io.IOException: Unable to complete flush {ENCODED => 744bbec6934be304e01b009be760323d, NAME => 'TestAcidGuarantees,,1732486556329.744bbec6934be304e01b009be760323d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T22:16:03,579 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bd6592c37bb2:0-0 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=126 java.io.IOException: Unable to complete flush {ENCODED => 744bbec6934be304e01b009be760323d, NAME => 'TestAcidGuarantees,,1732486556329.744bbec6934be304e01b009be760323d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T22:16:03,579 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37701 {}] master.HMaster(4177): Remote procedure failed, pid=126 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 744bbec6934be304e01b009be760323d, NAME => 'TestAcidGuarantees,,1732486556329.744bbec6934be304e01b009be760323d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 744bbec6934be304e01b009be760323d, NAME => 'TestAcidGuarantees,,1732486556329.744bbec6934be304e01b009be760323d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T22:16:03,585 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40087 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=744bbec6934be304e01b009be760323d, server=bd6592c37bb2,40087,1732486423713 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-24T22:16:03,585 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40087 {}] ipc.CallRunner(138): callId: 20 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46842 deadline: 1732486623580, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=744bbec6934be304e01b009be760323d, server=bd6592c37bb2,40087,1732486423713 2024-11-24T22:16:03,585 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40087 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=744bbec6934be304e01b009be760323d, server=bd6592c37bb2,40087,1732486423713 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-24T22:16:03,585 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732486556329.744bbec6934be304e01b009be760323d., hostname=bd6592c37bb2,40087,1732486423713, seqNum=2 , the old value is region=TestAcidGuarantees,,1732486556329.744bbec6934be304e01b009be760323d., hostname=bd6592c37bb2,40087,1732486423713, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=744bbec6934be304e01b009be760323d, server=bd6592c37bb2,40087,1732486423713 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T22:16:03,585 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40087 {}] ipc.CallRunner(138): callId: 20 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46816 deadline: 1732486623581, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=744bbec6934be304e01b009be760323d, server=bd6592c37bb2,40087,1732486423713 2024-11-24T22:16:03,585 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732486556329.744bbec6934be304e01b009be760323d., hostname=bd6592c37bb2,40087,1732486423713, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=744bbec6934be304e01b009be760323d, server=bd6592c37bb2,40087,1732486423713 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T22:16:03,585 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732486556329.744bbec6934be304e01b009be760323d., hostname=bd6592c37bb2,40087,1732486423713, seqNum=2 because the exception is null or not the one we care about 2024-11-24T22:16:03,586 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40087 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=744bbec6934be304e01b009be760323d, server=bd6592c37bb2,40087,1732486423713 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-24T22:16:03,586 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732486556329.744bbec6934be304e01b009be760323d., hostname=bd6592c37bb2,40087,1732486423713, seqNum=2 , the old value is region=TestAcidGuarantees,,1732486556329.744bbec6934be304e01b009be760323d., hostname=bd6592c37bb2,40087,1732486423713, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=744bbec6934be304e01b009be760323d, server=bd6592c37bb2,40087,1732486423713 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T22:16:03,586 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732486556329.744bbec6934be304e01b009be760323d., hostname=bd6592c37bb2,40087,1732486423713, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=744bbec6934be304e01b009be760323d, server=bd6592c37bb2,40087,1732486423713 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T22:16:03,586 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40087 {}] ipc.CallRunner(138): callId: 23 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46824 deadline: 1732486623581, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=744bbec6934be304e01b009be760323d, server=bd6592c37bb2,40087,1732486423713 2024-11-24T22:16:03,586 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732486556329.744bbec6934be304e01b009be760323d., hostname=bd6592c37bb2,40087,1732486423713, seqNum=2 because the exception is null or not the one we care about 2024-11-24T22:16:03,586 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732486556329.744bbec6934be304e01b009be760323d., hostname=bd6592c37bb2,40087,1732486423713, seqNum=2 , the old value is region=TestAcidGuarantees,,1732486556329.744bbec6934be304e01b009be760323d., hostname=bd6592c37bb2,40087,1732486423713, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=744bbec6934be304e01b009be760323d, server=bd6592c37bb2,40087,1732486423713 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T22:16:03,586 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732486556329.744bbec6934be304e01b009be760323d., hostname=bd6592c37bb2,40087,1732486423713, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=744bbec6934be304e01b009be760323d, server=bd6592c37bb2,40087,1732486423713 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T22:16:03,586 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40087 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=744bbec6934be304e01b009be760323d, server=bd6592c37bb2,40087,1732486423713 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-24T22:16:03,586 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732486556329.744bbec6934be304e01b009be760323d., hostname=bd6592c37bb2,40087,1732486423713, seqNum=2 because the exception is null or not the one we care about 2024-11-24T22:16:03,586 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40087 {}] ipc.CallRunner(138): callId: 21 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46820 deadline: 1732486623581, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=744bbec6934be304e01b009be760323d, server=bd6592c37bb2,40087,1732486423713 2024-11-24T22:16:03,586 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732486556329.744bbec6934be304e01b009be760323d., hostname=bd6592c37bb2,40087,1732486423713, seqNum=2 , the old value is region=TestAcidGuarantees,,1732486556329.744bbec6934be304e01b009be760323d., hostname=bd6592c37bb2,40087,1732486423713, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=744bbec6934be304e01b009be760323d, server=bd6592c37bb2,40087,1732486423713 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T22:16:03,586 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732486556329.744bbec6934be304e01b009be760323d., hostname=bd6592c37bb2,40087,1732486423713, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=744bbec6934be304e01b009be760323d, server=bd6592c37bb2,40087,1732486423713 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T22:16:03,586 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732486556329.744bbec6934be304e01b009be760323d., hostname=bd6592c37bb2,40087,1732486423713, seqNum=2 because the exception is null or not the one we care about 2024-11-24T22:16:03,613 DEBUG [RS:0;bd6592c37bb2:40087-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/744bbec6934be304e01b009be760323d/.tmp/C/dfd78c5f1e3446ceb9e6877167ddb6c0 as hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/744bbec6934be304e01b009be760323d/C/dfd78c5f1e3446ceb9e6877167ddb6c0 2024-11-24T22:16:03,617 INFO [RS:0;bd6592c37bb2:40087-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 744bbec6934be304e01b009be760323d/C of 744bbec6934be304e01b009be760323d into dfd78c5f1e3446ceb9e6877167ddb6c0(size=11.9 K), total size for store is 11.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-24T22:16:03,617 DEBUG [RS:0;bd6592c37bb2:40087-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 744bbec6934be304e01b009be760323d: 2024-11-24T22:16:03,617 INFO [RS:0;bd6592c37bb2:40087-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732486556329.744bbec6934be304e01b009be760323d., storeName=744bbec6934be304e01b009be760323d/C, priority=13, startTime=1732486562773; duration=0sec 2024-11-24T22:16:03,617 DEBUG [RS:0;bd6592c37bb2:40087-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-24T22:16:03,617 DEBUG [RS:0;bd6592c37bb2:40087-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 744bbec6934be304e01b009be760323d:C 2024-11-24T22:16:03,694 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=38.02 KB at sequenceid=115 (bloomFilter=true), to=hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/744bbec6934be304e01b009be760323d/.tmp/B/04e097aec304409694fd69e1932b2a7b 2024-11-24T22:16:03,700 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/744bbec6934be304e01b009be760323d/.tmp/C/a7b5aae7560f43f3af42b146d01b7d87 is 50, key is test_row_0/C:col10/1732486562047/Put/seqid=0 2024-11-24T22:16:03,703 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45929 is added to blk_1073742301_1477 (size=12001) 2024-11-24T22:16:03,731 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=40087 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=126 2024-11-24T22:16:03,731 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bd6592c37bb2:0-1 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732486556329.744bbec6934be304e01b009be760323d. 2024-11-24T22:16:03,731 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bd6592c37bb2:0-1 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732486556329.744bbec6934be304e01b009be760323d. as already flushing 2024-11-24T22:16:03,731 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bd6592c37bb2:0-1 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732486556329.744bbec6934be304e01b009be760323d. 2024-11-24T22:16:03,731 ERROR [RS_FLUSH_OPERATIONS-regionserver/bd6592c37bb2:0-1 {event_type=RS_FLUSH_REGIONS, pid=126}] handler.RSProcedureHandler(58): pid=126 java.io.IOException: Unable to complete flush {ENCODED => 744bbec6934be304e01b009be760323d, NAME => 'TestAcidGuarantees,,1732486556329.744bbec6934be304e01b009be760323d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T22:16:03,732 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bd6592c37bb2:0-1 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=126 java.io.IOException: Unable to complete flush {ENCODED => 744bbec6934be304e01b009be760323d, NAME => 'TestAcidGuarantees,,1732486556329.744bbec6934be304e01b009be760323d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T22:16:03,732 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37701 {}] master.HMaster(4177): Remote procedure failed, pid=126 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 744bbec6934be304e01b009be760323d, NAME => 'TestAcidGuarantees,,1732486556329.744bbec6934be304e01b009be760323d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 744bbec6934be304e01b009be760323d, NAME => 'TestAcidGuarantees,,1732486556329.744bbec6934be304e01b009be760323d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T22:16:03,884 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=40087 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=126 2024-11-24T22:16:03,884 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bd6592c37bb2:0-2 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732486556329.744bbec6934be304e01b009be760323d. 2024-11-24T22:16:03,884 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bd6592c37bb2:0-2 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732486556329.744bbec6934be304e01b009be760323d. as already flushing 2024-11-24T22:16:03,884 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bd6592c37bb2:0-2 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732486556329.744bbec6934be304e01b009be760323d. 2024-11-24T22:16:03,884 ERROR [RS_FLUSH_OPERATIONS-regionserver/bd6592c37bb2:0-2 {event_type=RS_FLUSH_REGIONS, pid=126}] handler.RSProcedureHandler(58): pid=126 java.io.IOException: Unable to complete flush {ENCODED => 744bbec6934be304e01b009be760323d, NAME => 'TestAcidGuarantees,,1732486556329.744bbec6934be304e01b009be760323d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T22:16:03,884 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bd6592c37bb2:0-2 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=126 java.io.IOException: Unable to complete flush {ENCODED => 744bbec6934be304e01b009be760323d, NAME => 'TestAcidGuarantees,,1732486556329.744bbec6934be304e01b009be760323d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T22:16:03,885 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37701 {}] master.HMaster(4177): Remote procedure failed, pid=126 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 744bbec6934be304e01b009be760323d, NAME => 'TestAcidGuarantees,,1732486556329.744bbec6934be304e01b009be760323d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 744bbec6934be304e01b009be760323d, NAME => 'TestAcidGuarantees,,1732486556329.744bbec6934be304e01b009be760323d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T22:16:03,950 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37701 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=125 2024-11-24T22:16:04,037 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=40087 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=126 2024-11-24T22:16:04,037 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bd6592c37bb2:0-0 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732486556329.744bbec6934be304e01b009be760323d. 2024-11-24T22:16:04,037 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bd6592c37bb2:0-0 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732486556329.744bbec6934be304e01b009be760323d. as already flushing 2024-11-24T22:16:04,037 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bd6592c37bb2:0-0 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732486556329.744bbec6934be304e01b009be760323d. 2024-11-24T22:16:04,037 ERROR [RS_FLUSH_OPERATIONS-regionserver/bd6592c37bb2:0-0 {event_type=RS_FLUSH_REGIONS, pid=126}] handler.RSProcedureHandler(58): pid=126 java.io.IOException: Unable to complete flush {ENCODED => 744bbec6934be304e01b009be760323d, NAME => 'TestAcidGuarantees,,1732486556329.744bbec6934be304e01b009be760323d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T22:16:04,037 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bd6592c37bb2:0-0 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=126 java.io.IOException: Unable to complete flush {ENCODED => 744bbec6934be304e01b009be760323d, NAME => 'TestAcidGuarantees,,1732486556329.744bbec6934be304e01b009be760323d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T22:16:04,038 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37701 {}] master.HMaster(4177): Remote procedure failed, pid=126 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 744bbec6934be304e01b009be760323d, NAME => 'TestAcidGuarantees,,1732486556329.744bbec6934be304e01b009be760323d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 744bbec6934be304e01b009be760323d, NAME => 'TestAcidGuarantees,,1732486556329.744bbec6934be304e01b009be760323d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T22:16:04,098 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40087 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=744bbec6934be304e01b009be760323d, server=bd6592c37bb2,40087,1732486423713 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-24T22:16:04,098 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40087 {}] ipc.CallRunner(138): callId: 21 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46842 deadline: 1732486624090, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=744bbec6934be304e01b009be760323d, server=bd6592c37bb2,40087,1732486423713 2024-11-24T22:16:04,099 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40087 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=744bbec6934be304e01b009be760323d, server=bd6592c37bb2,40087,1732486423713 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-24T22:16:04,099 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732486556329.744bbec6934be304e01b009be760323d., hostname=bd6592c37bb2,40087,1732486423713, seqNum=2 , the old value is region=TestAcidGuarantees,,1732486556329.744bbec6934be304e01b009be760323d., hostname=bd6592c37bb2,40087,1732486423713, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=744bbec6934be304e01b009be760323d, server=bd6592c37bb2,40087,1732486423713 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T22:16:04,099 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732486556329.744bbec6934be304e01b009be760323d., hostname=bd6592c37bb2,40087,1732486423713, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=744bbec6934be304e01b009be760323d, server=bd6592c37bb2,40087,1732486423713 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T22:16:04,099 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40087 {}] ipc.CallRunner(138): callId: 21 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46816 deadline: 1732486624091, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=744bbec6934be304e01b009be760323d, server=bd6592c37bb2,40087,1732486423713 2024-11-24T22:16:04,099 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732486556329.744bbec6934be304e01b009be760323d., hostname=bd6592c37bb2,40087,1732486423713, seqNum=2 because the exception is null or not the one we care about 2024-11-24T22:16:04,099 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40087 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=744bbec6934be304e01b009be760323d, server=bd6592c37bb2,40087,1732486423713 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-24T22:16:04,099 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732486556329.744bbec6934be304e01b009be760323d., hostname=bd6592c37bb2,40087,1732486423713, seqNum=2 , the old value is region=TestAcidGuarantees,,1732486556329.744bbec6934be304e01b009be760323d., hostname=bd6592c37bb2,40087,1732486423713, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=744bbec6934be304e01b009be760323d, server=bd6592c37bb2,40087,1732486423713 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T22:16:04,099 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732486556329.744bbec6934be304e01b009be760323d., hostname=bd6592c37bb2,40087,1732486423713, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=744bbec6934be304e01b009be760323d, server=bd6592c37bb2,40087,1732486423713 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T22:16:04,099 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40087 {}] ipc.CallRunner(138): callId: 24 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46824 deadline: 1732486624091, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=744bbec6934be304e01b009be760323d, server=bd6592c37bb2,40087,1732486423713 2024-11-24T22:16:04,099 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732486556329.744bbec6934be304e01b009be760323d., hostname=bd6592c37bb2,40087,1732486423713, seqNum=2 because the exception is null or not the one we care about 2024-11-24T22:16:04,099 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40087 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=744bbec6934be304e01b009be760323d, server=bd6592c37bb2,40087,1732486423713 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-24T22:16:04,099 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732486556329.744bbec6934be304e01b009be760323d., hostname=bd6592c37bb2,40087,1732486423713, seqNum=2 , the old value is region=TestAcidGuarantees,,1732486556329.744bbec6934be304e01b009be760323d., hostname=bd6592c37bb2,40087,1732486423713, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=744bbec6934be304e01b009be760323d, server=bd6592c37bb2,40087,1732486423713 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T22:16:04,099 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40087 {}] ipc.CallRunner(138): callId: 22 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46820 deadline: 1732486624091, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=744bbec6934be304e01b009be760323d, server=bd6592c37bb2,40087,1732486423713 2024-11-24T22:16:04,099 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732486556329.744bbec6934be304e01b009be760323d., hostname=bd6592c37bb2,40087,1732486423713, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=744bbec6934be304e01b009be760323d, server=bd6592c37bb2,40087,1732486423713 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T22:16:04,100 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732486556329.744bbec6934be304e01b009be760323d., hostname=bd6592c37bb2,40087,1732486423713, seqNum=2 because the exception is null or not the one we care about 2024-11-24T22:16:04,100 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732486556329.744bbec6934be304e01b009be760323d., hostname=bd6592c37bb2,40087,1732486423713, seqNum=2 , the old value is region=TestAcidGuarantees,,1732486556329.744bbec6934be304e01b009be760323d., hostname=bd6592c37bb2,40087,1732486423713, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=744bbec6934be304e01b009be760323d, server=bd6592c37bb2,40087,1732486423713 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T22:16:04,100 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732486556329.744bbec6934be304e01b009be760323d., hostname=bd6592c37bb2,40087,1732486423713, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=744bbec6934be304e01b009be760323d, server=bd6592c37bb2,40087,1732486423713 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T22:16:04,100 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732486556329.744bbec6934be304e01b009be760323d., hostname=bd6592c37bb2,40087,1732486423713, seqNum=2 because the exception is null or not the one we care about 2024-11-24T22:16:04,104 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=38.02 KB at sequenceid=115 (bloomFilter=true), to=hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/744bbec6934be304e01b009be760323d/.tmp/C/a7b5aae7560f43f3af42b146d01b7d87 2024-11-24T22:16:04,107 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/744bbec6934be304e01b009be760323d/.tmp/A/b78790bd2d7849b18fe9b44f38b57ece as hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/744bbec6934be304e01b009be760323d/A/b78790bd2d7849b18fe9b44f38b57ece 2024-11-24T22:16:04,111 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/744bbec6934be304e01b009be760323d/A/b78790bd2d7849b18fe9b44f38b57ece, entries=200, sequenceid=115, filesize=14.0 K 2024-11-24T22:16:04,111 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/744bbec6934be304e01b009be760323d/.tmp/B/04e097aec304409694fd69e1932b2a7b as hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/744bbec6934be304e01b009be760323d/B/04e097aec304409694fd69e1932b2a7b 2024-11-24T22:16:04,123 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/744bbec6934be304e01b009be760323d/B/04e097aec304409694fd69e1932b2a7b, entries=150, sequenceid=115, filesize=11.7 K 2024-11-24T22:16:04,124 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/744bbec6934be304e01b009be760323d/.tmp/C/a7b5aae7560f43f3af42b146d01b7d87 as hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/744bbec6934be304e01b009be760323d/C/a7b5aae7560f43f3af42b146d01b7d87 2024-11-24T22:16:04,127 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/744bbec6934be304e01b009be760323d/C/a7b5aae7560f43f3af42b146d01b7d87, entries=150, sequenceid=115, filesize=11.7 K 2024-11-24T22:16:04,128 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~114.05 KB/116790, heapSize ~339.38 KB/347520, currentSize=60.38 KB/61830 for 744bbec6934be304e01b009be760323d in 1255ms, sequenceid=115, compaction requested=false 2024-11-24T22:16:04,128 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 744bbec6934be304e01b009be760323d: 2024-11-24T22:16:04,189 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=40087 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=126 2024-11-24T22:16:04,190 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bd6592c37bb2:0-1 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732486556329.744bbec6934be304e01b009be760323d. 2024-11-24T22:16:04,190 INFO [RS_FLUSH_OPERATIONS-regionserver/bd6592c37bb2:0-1 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.HRegion(2902): Flushing 744bbec6934be304e01b009be760323d 3/3 column families, dataSize=60.38 KB heapSize=180.05 KB 2024-11-24T22:16:04,190 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bd6592c37bb2:0-1 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 744bbec6934be304e01b009be760323d, store=A 2024-11-24T22:16:04,190 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bd6592c37bb2:0-1 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-24T22:16:04,190 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bd6592c37bb2:0-1 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 744bbec6934be304e01b009be760323d, store=B 2024-11-24T22:16:04,190 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bd6592c37bb2:0-1 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-24T22:16:04,190 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bd6592c37bb2:0-1 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 744bbec6934be304e01b009be760323d, store=C 2024-11-24T22:16:04,190 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bd6592c37bb2:0-1 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-24T22:16:04,195 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bd6592c37bb2:0-1 {event_type=RS_FLUSH_REGIONS, pid=126}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/744bbec6934be304e01b009be760323d/.tmp/A/4678adece5b54edb9f8cb6392e04d599 is 50, key is test_row_0/A:col10/1732486562931/Put/seqid=0 2024-11-24T22:16:04,198 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45929 is added to blk_1073742302_1478 (size=12001) 2024-11-24T22:16:04,255 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40087 {}] regionserver.HRegion(8855): Flush requested on 744bbec6934be304e01b009be760323d 2024-11-24T22:16:04,255 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732486556329.744bbec6934be304e01b009be760323d. as already flushing 2024-11-24T22:16:04,389 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40087 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=744bbec6934be304e01b009be760323d, server=bd6592c37bb2,40087,1732486423713 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-24T22:16:04,390 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40087 {}] ipc.CallRunner(138): callId: 93 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46792 deadline: 1732486624380, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=744bbec6934be304e01b009be760323d, server=bd6592c37bb2,40087,1732486423713 2024-11-24T22:16:04,390 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732486556329.744bbec6934be304e01b009be760323d., hostname=bd6592c37bb2,40087,1732486423713, seqNum=2 , the old value is region=TestAcidGuarantees,,1732486556329.744bbec6934be304e01b009be760323d., hostname=bd6592c37bb2,40087,1732486423713, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=744bbec6934be304e01b009be760323d, server=bd6592c37bb2,40087,1732486423713 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T22:16:04,390 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732486556329.744bbec6934be304e01b009be760323d., hostname=bd6592c37bb2,40087,1732486423713, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=744bbec6934be304e01b009be760323d, server=bd6592c37bb2,40087,1732486423713 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T22:16:04,390 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732486556329.744bbec6934be304e01b009be760323d., hostname=bd6592c37bb2,40087,1732486423713, seqNum=2 because the exception is null or not the one we care about 2024-11-24T22:16:04,508 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40087 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=744bbec6934be304e01b009be760323d, server=bd6592c37bb2,40087,1732486423713 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-24T22:16:04,509 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40087 {}] ipc.CallRunner(138): callId: 94 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46792 deadline: 1732486624501, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=744bbec6934be304e01b009be760323d, server=bd6592c37bb2,40087,1732486423713 2024-11-24T22:16:04,509 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732486556329.744bbec6934be304e01b009be760323d., hostname=bd6592c37bb2,40087,1732486423713, seqNum=2 , the old value is region=TestAcidGuarantees,,1732486556329.744bbec6934be304e01b009be760323d., hostname=bd6592c37bb2,40087,1732486423713, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=744bbec6934be304e01b009be760323d, server=bd6592c37bb2,40087,1732486423713 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T22:16:04,509 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732486556329.744bbec6934be304e01b009be760323d., hostname=bd6592c37bb2,40087,1732486423713, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=744bbec6934be304e01b009be760323d, server=bd6592c37bb2,40087,1732486423713 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T22:16:04,509 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732486556329.744bbec6934be304e01b009be760323d., hostname=bd6592c37bb2,40087,1732486423713, seqNum=2 because the exception is null or not the one we care about 2024-11-24T22:16:04,599 INFO [RS_FLUSH_OPERATIONS-regionserver/bd6592c37bb2:0-1 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=130 (bloomFilter=true), to=hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/744bbec6934be304e01b009be760323d/.tmp/A/4678adece5b54edb9f8cb6392e04d599 2024-11-24T22:16:04,605 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bd6592c37bb2:0-1 {event_type=RS_FLUSH_REGIONS, pid=126}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/744bbec6934be304e01b009be760323d/.tmp/B/d202f49d4dfd4adcacb0e187b0631626 is 50, key is test_row_0/B:col10/1732486562931/Put/seqid=0 2024-11-24T22:16:04,609 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45929 is added to blk_1073742303_1479 (size=12001) 2024-11-24T22:16:04,610 INFO [RS_FLUSH_OPERATIONS-regionserver/bd6592c37bb2:0-1 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=130 (bloomFilter=true), to=hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/744bbec6934be304e01b009be760323d/.tmp/B/d202f49d4dfd4adcacb0e187b0631626 2024-11-24T22:16:04,616 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bd6592c37bb2:0-1 {event_type=RS_FLUSH_REGIONS, pid=126}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/744bbec6934be304e01b009be760323d/.tmp/C/0ad322a0749345fdb793029953abefd7 is 50, key is test_row_0/C:col10/1732486562931/Put/seqid=0 2024-11-24T22:16:04,619 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45929 is added to blk_1073742304_1480 (size=12001) 2024-11-24T22:16:04,732 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40087 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=744bbec6934be304e01b009be760323d, server=bd6592c37bb2,40087,1732486423713 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-24T22:16:04,733 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40087 {}] ipc.CallRunner(138): callId: 95 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46792 deadline: 1732486624720, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=744bbec6934be304e01b009be760323d, server=bd6592c37bb2,40087,1732486423713 2024-11-24T22:16:04,733 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732486556329.744bbec6934be304e01b009be760323d., hostname=bd6592c37bb2,40087,1732486423713, seqNum=2 , the old value is region=TestAcidGuarantees,,1732486556329.744bbec6934be304e01b009be760323d., hostname=bd6592c37bb2,40087,1732486423713, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=744bbec6934be304e01b009be760323d, server=bd6592c37bb2,40087,1732486423713 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T22:16:04,733 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732486556329.744bbec6934be304e01b009be760323d., hostname=bd6592c37bb2,40087,1732486423713, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=744bbec6934be304e01b009be760323d, server=bd6592c37bb2,40087,1732486423713 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T22:16:04,733 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732486556329.744bbec6934be304e01b009be760323d., hostname=bd6592c37bb2,40087,1732486423713, seqNum=2 because the exception is null or not the one we care about 2024-11-24T22:16:04,961 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37701 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=125 2024-11-24T22:16:05,020 INFO [RS_FLUSH_OPERATIONS-regionserver/bd6592c37bb2:0-1 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=130 (bloomFilter=true), to=hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/744bbec6934be304e01b009be760323d/.tmp/C/0ad322a0749345fdb793029953abefd7 2024-11-24T22:16:05,024 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bd6592c37bb2:0-1 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/744bbec6934be304e01b009be760323d/.tmp/A/4678adece5b54edb9f8cb6392e04d599 as hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/744bbec6934be304e01b009be760323d/A/4678adece5b54edb9f8cb6392e04d599 2024-11-24T22:16:05,028 INFO [RS_FLUSH_OPERATIONS-regionserver/bd6592c37bb2:0-1 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/744bbec6934be304e01b009be760323d/A/4678adece5b54edb9f8cb6392e04d599, entries=150, sequenceid=130, filesize=11.7 K 2024-11-24T22:16:05,028 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bd6592c37bb2:0-1 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.StoreScanner(997): StoreScanner already has the close lock. There is no need to updateReaders 2024-11-24T22:16:05,029 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bd6592c37bb2:0-1 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/744bbec6934be304e01b009be760323d/.tmp/B/d202f49d4dfd4adcacb0e187b0631626 as hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/744bbec6934be304e01b009be760323d/B/d202f49d4dfd4adcacb0e187b0631626 2024-11-24T22:16:05,033 INFO [RS_FLUSH_OPERATIONS-regionserver/bd6592c37bb2:0-1 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/744bbec6934be304e01b009be760323d/B/d202f49d4dfd4adcacb0e187b0631626, entries=150, sequenceid=130, filesize=11.7 K 2024-11-24T22:16:05,033 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bd6592c37bb2:0-1 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/744bbec6934be304e01b009be760323d/.tmp/C/0ad322a0749345fdb793029953abefd7 as hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/744bbec6934be304e01b009be760323d/C/0ad322a0749345fdb793029953abefd7 2024-11-24T22:16:05,036 INFO [RS_FLUSH_OPERATIONS-regionserver/bd6592c37bb2:0-1 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/744bbec6934be304e01b009be760323d/C/0ad322a0749345fdb793029953abefd7, entries=150, sequenceid=130, filesize=11.7 K 2024-11-24T22:16:05,037 INFO [RS_FLUSH_OPERATIONS-regionserver/bd6592c37bb2:0-1 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.HRegion(3140): Finished flush of dataSize ~60.38 KB/61830, heapSize ~180 KB/184320, currentSize=114.05 KB/116790 for 744bbec6934be304e01b009be760323d in 847ms, sequenceid=130, compaction requested=true 2024-11-24T22:16:05,037 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bd6592c37bb2:0-1 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.HRegion(2603): Flush status journal for 744bbec6934be304e01b009be760323d: 2024-11-24T22:16:05,037 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bd6592c37bb2:0-1 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732486556329.744bbec6934be304e01b009be760323d. 2024-11-24T22:16:05,037 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bd6592c37bb2:0-1 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=126 2024-11-24T22:16:05,040 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37701 {}] master.HMaster(4169): Remote procedure done, pid=126 2024-11-24T22:16:05,043 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40087 {}] regionserver.HRegion(8855): Flush requested on 744bbec6934be304e01b009be760323d 2024-11-24T22:16:05,043 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 744bbec6934be304e01b009be760323d 3/3 column families, dataSize=120.76 KB heapSize=360.05 KB 2024-11-24T22:16:05,044 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 744bbec6934be304e01b009be760323d, store=A 2024-11-24T22:16:05,044 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-24T22:16:05,044 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 744bbec6934be304e01b009be760323d, store=B 2024-11-24T22:16:05,044 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-24T22:16:05,044 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 744bbec6934be304e01b009be760323d, store=C 2024-11-24T22:16:05,044 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-24T22:16:05,046 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=126, resume processing ppid=125 2024-11-24T22:16:05,046 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=126, ppid=125, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 2.2280 sec 2024-11-24T22:16:05,048 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=125, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=125, table=TestAcidGuarantees in 2.2340 sec 2024-11-24T22:16:05,049 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/744bbec6934be304e01b009be760323d/.tmp/A/488b263258da47168619dd5d260b7365 is 50, key is test_row_0/A:col10/1732486565041/Put/seqid=0 2024-11-24T22:16:05,053 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45929 is added to blk_1073742305_1481 (size=14541) 2024-11-24T22:16:05,119 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40087 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=744bbec6934be304e01b009be760323d, server=bd6592c37bb2,40087,1732486423713 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-24T22:16:05,119 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40087 {}] ipc.CallRunner(138): callId: 22 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46816 deadline: 1732486625111, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=744bbec6934be304e01b009be760323d, server=bd6592c37bb2,40087,1732486423713 2024-11-24T22:16:05,119 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40087 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=744bbec6934be304e01b009be760323d, server=bd6592c37bb2,40087,1732486423713 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-24T22:16:05,120 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40087 {}] ipc.CallRunner(138): callId: 25 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46824 deadline: 1732486625111, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=744bbec6934be304e01b009be760323d, server=bd6592c37bb2,40087,1732486423713 2024-11-24T22:16:05,120 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40087 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=744bbec6934be304e01b009be760323d, server=bd6592c37bb2,40087,1732486423713 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-24T22:16:05,120 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40087 {}] ipc.CallRunner(138): callId: 23 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46820 deadline: 1732486625111, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=744bbec6934be304e01b009be760323d, server=bd6592c37bb2,40087,1732486423713 2024-11-24T22:16:05,120 WARN [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRpcRetryingCaller(177): Call to bd6592c37bb2,40087,1732486423713 for 'test_row_2' in 744bbec6934be304e01b009be760323d of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2191 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=744bbec6934be304e01b009be760323d, server=bd6592c37bb2,40087,1732486423713 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor44.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=744bbec6934be304e01b009be760323d, server=bd6592c37bb2,40087,1732486423713 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-24T22:16:05,120 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40087 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=744bbec6934be304e01b009be760323d, server=bd6592c37bb2,40087,1732486423713 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-24T22:16:05,120 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732486556329.744bbec6934be304e01b009be760323d., hostname=bd6592c37bb2,40087,1732486423713, seqNum=2 , the old value is region=TestAcidGuarantees,,1732486556329.744bbec6934be304e01b009be760323d., hostname=bd6592c37bb2,40087,1732486423713, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=744bbec6934be304e01b009be760323d, server=bd6592c37bb2,40087,1732486423713 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T22:16:05,120 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732486556329.744bbec6934be304e01b009be760323d., hostname=bd6592c37bb2,40087,1732486423713, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=744bbec6934be304e01b009be760323d, server=bd6592c37bb2,40087,1732486423713 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T22:16:05,120 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40087 {}] ipc.CallRunner(138): callId: 104 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46792 deadline: 1732486625114, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=744bbec6934be304e01b009be760323d, server=bd6592c37bb2,40087,1732486423713 2024-11-24T22:16:05,120 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732486556329.744bbec6934be304e01b009be760323d., hostname=bd6592c37bb2,40087,1732486423713, seqNum=2 because the exception is null or not the one we care about 2024-11-24T22:16:05,120 WARN [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRpcRetryingCaller(177): Call to bd6592c37bb2,40087,1732486423713 for 'test_row_0' in 744bbec6934be304e01b009be760323d of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2187 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=744bbec6934be304e01b009be760323d, server=bd6592c37bb2,40087,1732486423713 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor44.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=744bbec6934be304e01b009be760323d, server=bd6592c37bb2,40087,1732486423713 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-24T22:16:05,120 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732486556329.744bbec6934be304e01b009be760323d., hostname=bd6592c37bb2,40087,1732486423713, seqNum=2 , the old value is region=TestAcidGuarantees,,1732486556329.744bbec6934be304e01b009be760323d., hostname=bd6592c37bb2,40087,1732486423713, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=744bbec6934be304e01b009be760323d, server=bd6592c37bb2,40087,1732486423713 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T22:16:05,121 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732486556329.744bbec6934be304e01b009be760323d., hostname=bd6592c37bb2,40087,1732486423713, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=744bbec6934be304e01b009be760323d, server=bd6592c37bb2,40087,1732486423713 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T22:16:05,120 WARN [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRpcRetryingCaller(177): Call to bd6592c37bb2,40087,1732486423713 for 'test_row_0' in 744bbec6934be304e01b009be760323d of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2187 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=744bbec6934be304e01b009be760323d, server=bd6592c37bb2,40087,1732486423713 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor44.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=744bbec6934be304e01b009be760323d, server=bd6592c37bb2,40087,1732486423713 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-24T22:16:05,121 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732486556329.744bbec6934be304e01b009be760323d., hostname=bd6592c37bb2,40087,1732486423713, seqNum=2 , the old value is region=TestAcidGuarantees,,1732486556329.744bbec6934be304e01b009be760323d., hostname=bd6592c37bb2,40087,1732486423713, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=744bbec6934be304e01b009be760323d, server=bd6592c37bb2,40087,1732486423713 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T22:16:05,121 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732486556329.744bbec6934be304e01b009be760323d., hostname=bd6592c37bb2,40087,1732486423713, seqNum=2 because the exception is null or not the one we care about 2024-11-24T22:16:05,121 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732486556329.744bbec6934be304e01b009be760323d., hostname=bd6592c37bb2,40087,1732486423713, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=744bbec6934be304e01b009be760323d, server=bd6592c37bb2,40087,1732486423713 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T22:16:05,121 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732486556329.744bbec6934be304e01b009be760323d., hostname=bd6592c37bb2,40087,1732486423713, seqNum=2 , the old value is region=TestAcidGuarantees,,1732486556329.744bbec6934be304e01b009be760323d., hostname=bd6592c37bb2,40087,1732486423713, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=744bbec6934be304e01b009be760323d, server=bd6592c37bb2,40087,1732486423713 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T22:16:05,121 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732486556329.744bbec6934be304e01b009be760323d., hostname=bd6592c37bb2,40087,1732486423713, seqNum=2 because the exception is null or not the one we care about 2024-11-24T22:16:05,121 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732486556329.744bbec6934be304e01b009be760323d., hostname=bd6592c37bb2,40087,1732486423713, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=744bbec6934be304e01b009be760323d, server=bd6592c37bb2,40087,1732486423713 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T22:16:05,121 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732486556329.744bbec6934be304e01b009be760323d., hostname=bd6592c37bb2,40087,1732486423713, seqNum=2 because the exception is null or not the one we care about 2024-11-24T22:16:05,126 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40087 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=744bbec6934be304e01b009be760323d, server=bd6592c37bb2,40087,1732486423713 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-24T22:16:05,127 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40087 {}] ipc.CallRunner(138): callId: 23 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46842 deadline: 1732486625119, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=744bbec6934be304e01b009be760323d, server=bd6592c37bb2,40087,1732486423713 2024-11-24T22:16:05,127 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732486556329.744bbec6934be304e01b009be760323d., hostname=bd6592c37bb2,40087,1732486423713, seqNum=2 , the old value is region=TestAcidGuarantees,,1732486556329.744bbec6934be304e01b009be760323d., hostname=bd6592c37bb2,40087,1732486423713, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=744bbec6934be304e01b009be760323d, server=bd6592c37bb2,40087,1732486423713 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T22:16:05,127 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732486556329.744bbec6934be304e01b009be760323d., hostname=bd6592c37bb2,40087,1732486423713, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=744bbec6934be304e01b009be760323d, server=bd6592c37bb2,40087,1732486423713 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T22:16:05,127 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732486556329.744bbec6934be304e01b009be760323d., hostname=bd6592c37bb2,40087,1732486423713, seqNum=2 because the exception is null or not the one we care about 2024-11-24T22:16:05,241 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40087 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=744bbec6934be304e01b009be760323d, server=bd6592c37bb2,40087,1732486423713 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-24T22:16:05,242 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40087 {}] ipc.CallRunner(138): callId: 105 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46792 deadline: 1732486625231, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=744bbec6934be304e01b009be760323d, server=bd6592c37bb2,40087,1732486423713 2024-11-24T22:16:05,242 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732486556329.744bbec6934be304e01b009be760323d., hostname=bd6592c37bb2,40087,1732486423713, seqNum=2 , the old value is region=TestAcidGuarantees,,1732486556329.744bbec6934be304e01b009be760323d., hostname=bd6592c37bb2,40087,1732486423713, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=744bbec6934be304e01b009be760323d, server=bd6592c37bb2,40087,1732486423713 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T22:16:05,242 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40087 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=744bbec6934be304e01b009be760323d, server=bd6592c37bb2,40087,1732486423713 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-24T22:16:05,242 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732486556329.744bbec6934be304e01b009be760323d., hostname=bd6592c37bb2,40087,1732486423713, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=744bbec6934be304e01b009be760323d, server=bd6592c37bb2,40087,1732486423713 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T22:16:05,242 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732486556329.744bbec6934be304e01b009be760323d., hostname=bd6592c37bb2,40087,1732486423713, seqNum=2 because the exception is null or not the one we care about 2024-11-24T22:16:05,242 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40087 {}] ipc.CallRunner(138): callId: 24 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46842 deadline: 1732486625231, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=744bbec6934be304e01b009be760323d, server=bd6592c37bb2,40087,1732486423713 2024-11-24T22:16:05,242 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732486556329.744bbec6934be304e01b009be760323d., hostname=bd6592c37bb2,40087,1732486423713, seqNum=2 , the old value is region=TestAcidGuarantees,,1732486556329.744bbec6934be304e01b009be760323d., hostname=bd6592c37bb2,40087,1732486423713, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=744bbec6934be304e01b009be760323d, server=bd6592c37bb2,40087,1732486423713 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T22:16:05,243 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732486556329.744bbec6934be304e01b009be760323d., hostname=bd6592c37bb2,40087,1732486423713, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=744bbec6934be304e01b009be760323d, server=bd6592c37bb2,40087,1732486423713 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T22:16:05,243 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732486556329.744bbec6934be304e01b009be760323d., hostname=bd6592c37bb2,40087,1732486423713, seqNum=2 because the exception is null or not the one we care about 2024-11-24T22:16:05,454 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=40.25 KB at sequenceid=151 (bloomFilter=true), to=hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/744bbec6934be304e01b009be760323d/.tmp/A/488b263258da47168619dd5d260b7365 2024-11-24T22:16:05,455 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40087 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=744bbec6934be304e01b009be760323d, server=bd6592c37bb2,40087,1732486423713 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-24T22:16:05,456 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40087 {}] ipc.CallRunner(138): callId: 106 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46792 deadline: 1732486625451, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=744bbec6934be304e01b009be760323d, server=bd6592c37bb2,40087,1732486423713 2024-11-24T22:16:05,456 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40087 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=744bbec6934be304e01b009be760323d, server=bd6592c37bb2,40087,1732486423713 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-24T22:16:05,456 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732486556329.744bbec6934be304e01b009be760323d., hostname=bd6592c37bb2,40087,1732486423713, seqNum=2 , the old value is region=TestAcidGuarantees,,1732486556329.744bbec6934be304e01b009be760323d., hostname=bd6592c37bb2,40087,1732486423713, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=744bbec6934be304e01b009be760323d, server=bd6592c37bb2,40087,1732486423713 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T22:16:05,456 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732486556329.744bbec6934be304e01b009be760323d., hostname=bd6592c37bb2,40087,1732486423713, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=744bbec6934be304e01b009be760323d, server=bd6592c37bb2,40087,1732486423713 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T22:16:05,456 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732486556329.744bbec6934be304e01b009be760323d., hostname=bd6592c37bb2,40087,1732486423713, seqNum=2 because the exception is null or not the one we care about 2024-11-24T22:16:05,456 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40087 {}] ipc.CallRunner(138): callId: 25 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46842 deadline: 1732486625451, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=744bbec6934be304e01b009be760323d, server=bd6592c37bb2,40087,1732486423713 2024-11-24T22:16:05,456 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732486556329.744bbec6934be304e01b009be760323d., hostname=bd6592c37bb2,40087,1732486423713, seqNum=2 , the old value is region=TestAcidGuarantees,,1732486556329.744bbec6934be304e01b009be760323d., hostname=bd6592c37bb2,40087,1732486423713, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=744bbec6934be304e01b009be760323d, server=bd6592c37bb2,40087,1732486423713 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T22:16:05,456 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732486556329.744bbec6934be304e01b009be760323d., hostname=bd6592c37bb2,40087,1732486423713, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=744bbec6934be304e01b009be760323d, server=bd6592c37bb2,40087,1732486423713 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T22:16:05,456 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732486556329.744bbec6934be304e01b009be760323d., hostname=bd6592c37bb2,40087,1732486423713, seqNum=2 because the exception is null or not the one we care about 2024-11-24T22:16:05,461 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/744bbec6934be304e01b009be760323d/.tmp/B/ce895edc649449af9eabc9f2f406797a is 50, key is test_row_0/B:col10/1732486565041/Put/seqid=0 2024-11-24T22:16:05,464 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45929 is added to blk_1073742306_1482 (size=12151) 2024-11-24T22:16:05,764 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40087 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=744bbec6934be304e01b009be760323d, server=bd6592c37bb2,40087,1732486423713 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-24T22:16:05,764 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40087 {}] ipc.CallRunner(138): callId: 107 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46792 deadline: 1732486625760, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=744bbec6934be304e01b009be760323d, server=bd6592c37bb2,40087,1732486423713 2024-11-24T22:16:05,764 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40087 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=744bbec6934be304e01b009be760323d, server=bd6592c37bb2,40087,1732486423713 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-24T22:16:05,764 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732486556329.744bbec6934be304e01b009be760323d., hostname=bd6592c37bb2,40087,1732486423713, seqNum=2 , the old value is region=TestAcidGuarantees,,1732486556329.744bbec6934be304e01b009be760323d., hostname=bd6592c37bb2,40087,1732486423713, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=744bbec6934be304e01b009be760323d, server=bd6592c37bb2,40087,1732486423713 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T22:16:05,764 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40087 {}] ipc.CallRunner(138): callId: 26 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46842 deadline: 1732486625760, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=744bbec6934be304e01b009be760323d, server=bd6592c37bb2,40087,1732486423713 2024-11-24T22:16:05,764 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732486556329.744bbec6934be304e01b009be760323d., hostname=bd6592c37bb2,40087,1732486423713, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=744bbec6934be304e01b009be760323d, server=bd6592c37bb2,40087,1732486423713 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T22:16:05,765 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732486556329.744bbec6934be304e01b009be760323d., hostname=bd6592c37bb2,40087,1732486423713, seqNum=2 because the exception is null or not the one we care about 2024-11-24T22:16:05,765 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732486556329.744bbec6934be304e01b009be760323d., hostname=bd6592c37bb2,40087,1732486423713, seqNum=2 , the old value is region=TestAcidGuarantees,,1732486556329.744bbec6934be304e01b009be760323d., hostname=bd6592c37bb2,40087,1732486423713, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=744bbec6934be304e01b009be760323d, server=bd6592c37bb2,40087,1732486423713 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T22:16:05,765 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732486556329.744bbec6934be304e01b009be760323d., hostname=bd6592c37bb2,40087,1732486423713, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=744bbec6934be304e01b009be760323d, server=bd6592c37bb2,40087,1732486423713 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T22:16:05,765 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732486556329.744bbec6934be304e01b009be760323d., hostname=bd6592c37bb2,40087,1732486423713, seqNum=2 because the exception is null or not the one we care about 2024-11-24T22:16:05,865 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=40.25 KB at sequenceid=151 (bloomFilter=true), to=hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/744bbec6934be304e01b009be760323d/.tmp/B/ce895edc649449af9eabc9f2f406797a 2024-11-24T22:16:05,872 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/744bbec6934be304e01b009be760323d/.tmp/C/e04a440a3f514792b596de994ba37e38 is 50, key is test_row_0/C:col10/1732486565041/Put/seqid=0 2024-11-24T22:16:05,875 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45929 is added to blk_1073742307_1483 (size=12151) 2024-11-24T22:16:06,276 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=40.25 KB at sequenceid=151 (bloomFilter=true), to=hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/744bbec6934be304e01b009be760323d/.tmp/C/e04a440a3f514792b596de994ba37e38 2024-11-24T22:16:06,278 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40087 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=744bbec6934be304e01b009be760323d, server=bd6592c37bb2,40087,1732486423713 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-24T22:16:06,278 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40087 {}] ipc.CallRunner(138): callId: 108 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46792 deadline: 1732486626271, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=744bbec6934be304e01b009be760323d, server=bd6592c37bb2,40087,1732486423713 2024-11-24T22:16:06,278 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732486556329.744bbec6934be304e01b009be760323d., hostname=bd6592c37bb2,40087,1732486423713, seqNum=2 , the old value is region=TestAcidGuarantees,,1732486556329.744bbec6934be304e01b009be760323d., hostname=bd6592c37bb2,40087,1732486423713, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=744bbec6934be304e01b009be760323d, server=bd6592c37bb2,40087,1732486423713 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T22:16:06,278 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40087 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=744bbec6934be304e01b009be760323d, server=bd6592c37bb2,40087,1732486423713 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-24T22:16:06,278 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732486556329.744bbec6934be304e01b009be760323d., hostname=bd6592c37bb2,40087,1732486423713, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=744bbec6934be304e01b009be760323d, server=bd6592c37bb2,40087,1732486423713 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T22:16:06,278 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732486556329.744bbec6934be304e01b009be760323d., hostname=bd6592c37bb2,40087,1732486423713, seqNum=2 because the exception is null or not the one we care about 2024-11-24T22:16:06,278 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40087 {}] ipc.CallRunner(138): callId: 27 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46842 deadline: 1732486626271, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=744bbec6934be304e01b009be760323d, server=bd6592c37bb2,40087,1732486423713 2024-11-24T22:16:06,279 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732486556329.744bbec6934be304e01b009be760323d., hostname=bd6592c37bb2,40087,1732486423713, seqNum=2 , the old value is region=TestAcidGuarantees,,1732486556329.744bbec6934be304e01b009be760323d., hostname=bd6592c37bb2,40087,1732486423713, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=744bbec6934be304e01b009be760323d, server=bd6592c37bb2,40087,1732486423713 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T22:16:06,279 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732486556329.744bbec6934be304e01b009be760323d., hostname=bd6592c37bb2,40087,1732486423713, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=744bbec6934be304e01b009be760323d, server=bd6592c37bb2,40087,1732486423713 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T22:16:06,279 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732486556329.744bbec6934be304e01b009be760323d., hostname=bd6592c37bb2,40087,1732486423713, seqNum=2 because the exception is null or not the one we care about 2024-11-24T22:16:06,280 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/744bbec6934be304e01b009be760323d/.tmp/A/488b263258da47168619dd5d260b7365 as hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/744bbec6934be304e01b009be760323d/A/488b263258da47168619dd5d260b7365 2024-11-24T22:16:06,282 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/744bbec6934be304e01b009be760323d/A/488b263258da47168619dd5d260b7365, entries=200, sequenceid=151, filesize=14.2 K 2024-11-24T22:16:06,283 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/744bbec6934be304e01b009be760323d/.tmp/B/ce895edc649449af9eabc9f2f406797a as hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/744bbec6934be304e01b009be760323d/B/ce895edc649449af9eabc9f2f406797a 2024-11-24T22:16:06,286 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/744bbec6934be304e01b009be760323d/B/ce895edc649449af9eabc9f2f406797a, entries=150, sequenceid=151, filesize=11.9 K 2024-11-24T22:16:06,286 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/744bbec6934be304e01b009be760323d/.tmp/C/e04a440a3f514792b596de994ba37e38 as hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/744bbec6934be304e01b009be760323d/C/e04a440a3f514792b596de994ba37e38 2024-11-24T22:16:06,289 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/744bbec6934be304e01b009be760323d/C/e04a440a3f514792b596de994ba37e38, entries=150, sequenceid=151, filesize=11.9 K 2024-11-24T22:16:06,290 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~120.76 KB/123660, heapSize ~360 KB/368640, currentSize=53.67 KB/54960 for 744bbec6934be304e01b009be760323d in 1247ms, sequenceid=151, compaction requested=true 2024-11-24T22:16:06,290 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 744bbec6934be304e01b009be760323d: 2024-11-24T22:16:06,290 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 744bbec6934be304e01b009be760323d:A, priority=-2147483648, current under compaction store size is 1 2024-11-24T22:16:06,290 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-24T22:16:06,290 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 744bbec6934be304e01b009be760323d:B, priority=-2147483648, current under compaction store size is 2 2024-11-24T22:16:06,290 DEBUG [RS:0;bd6592c37bb2:40087-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-24T22:16:06,290 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-24T22:16:06,290 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 744bbec6934be304e01b009be760323d:C, priority=-2147483648, current under compaction store size is 3 2024-11-24T22:16:06,290 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-24T22:16:06,290 DEBUG [RS:0;bd6592c37bb2:40087-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-24T22:16:06,291 DEBUG [RS:0;bd6592c37bb2:40087-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 53090 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-24T22:16:06,291 DEBUG [RS:0;bd6592c37bb2:40087-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 48360 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-24T22:16:06,292 DEBUG [RS:0;bd6592c37bb2:40087-shortCompactions-0 {}] regionserver.HStore(1541): 744bbec6934be304e01b009be760323d/A is initiating minor compaction (all files) 2024-11-24T22:16:06,292 DEBUG [RS:0;bd6592c37bb2:40087-longCompactions-0 {}] regionserver.HStore(1541): 744bbec6934be304e01b009be760323d/B is initiating minor compaction (all files) 2024-11-24T22:16:06,292 INFO [RS:0;bd6592c37bb2:40087-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 744bbec6934be304e01b009be760323d/A in TestAcidGuarantees,,1732486556329.744bbec6934be304e01b009be760323d. 2024-11-24T22:16:06,292 INFO [RS:0;bd6592c37bb2:40087-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 744bbec6934be304e01b009be760323d/B in TestAcidGuarantees,,1732486556329.744bbec6934be304e01b009be760323d. 2024-11-24T22:16:06,292 INFO [RS:0;bd6592c37bb2:40087-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/744bbec6934be304e01b009be760323d/A/85a7f1a085d7453a819dea01d918cb9a, hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/744bbec6934be304e01b009be760323d/A/b78790bd2d7849b18fe9b44f38b57ece, hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/744bbec6934be304e01b009be760323d/A/4678adece5b54edb9f8cb6392e04d599, hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/744bbec6934be304e01b009be760323d/A/488b263258da47168619dd5d260b7365] into tmpdir=hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/744bbec6934be304e01b009be760323d/.tmp, totalSize=51.8 K 2024-11-24T22:16:06,292 INFO [RS:0;bd6592c37bb2:40087-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/744bbec6934be304e01b009be760323d/B/9b45215aecb6437aa1a20c2a2e6823e5, hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/744bbec6934be304e01b009be760323d/B/04e097aec304409694fd69e1932b2a7b, hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/744bbec6934be304e01b009be760323d/B/d202f49d4dfd4adcacb0e187b0631626, hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/744bbec6934be304e01b009be760323d/B/ce895edc649449af9eabc9f2f406797a] into tmpdir=hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/744bbec6934be304e01b009be760323d/.tmp, totalSize=47.2 K 2024-11-24T22:16:06,292 DEBUG [RS:0;bd6592c37bb2:40087-shortCompactions-0 {}] compactions.Compactor(225): Compacting 85a7f1a085d7453a819dea01d918cb9a, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=95, earliestPutTs=1732486561797 2024-11-24T22:16:06,292 DEBUG [RS:0;bd6592c37bb2:40087-longCompactions-0 {}] compactions.Compactor(225): Compacting 9b45215aecb6437aa1a20c2a2e6823e5, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=95, earliestPutTs=1732486561797 2024-11-24T22:16:06,292 DEBUG [RS:0;bd6592c37bb2:40087-longCompactions-0 {}] compactions.Compactor(225): Compacting 04e097aec304409694fd69e1932b2a7b, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=115, earliestPutTs=1732486562026 2024-11-24T22:16:06,292 DEBUG [RS:0;bd6592c37bb2:40087-shortCompactions-0 {}] compactions.Compactor(225): Compacting b78790bd2d7849b18fe9b44f38b57ece, keycount=200, bloomtype=ROW, size=14.0 K, encoding=NONE, compression=NONE, seqNum=115, earliestPutTs=1732486562026 2024-11-24T22:16:06,293 DEBUG [RS:0;bd6592c37bb2:40087-longCompactions-0 {}] compactions.Compactor(225): Compacting d202f49d4dfd4adcacb0e187b0631626, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=130, earliestPutTs=1732486562926 2024-11-24T22:16:06,293 DEBUG [RS:0;bd6592c37bb2:40087-shortCompactions-0 {}] compactions.Compactor(225): Compacting 4678adece5b54edb9f8cb6392e04d599, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=130, earliestPutTs=1732486562926 2024-11-24T22:16:06,293 DEBUG [RS:0;bd6592c37bb2:40087-shortCompactions-0 {}] compactions.Compactor(225): Compacting 488b263258da47168619dd5d260b7365, keycount=200, bloomtype=ROW, size=14.2 K, encoding=NONE, compression=NONE, seqNum=151, earliestPutTs=1732486564349 2024-11-24T22:16:06,293 DEBUG [RS:0;bd6592c37bb2:40087-longCompactions-0 {}] compactions.Compactor(225): Compacting ce895edc649449af9eabc9f2f406797a, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=151, earliestPutTs=1732486564371 2024-11-24T22:16:06,301 INFO [RS:0;bd6592c37bb2:40087-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 744bbec6934be304e01b009be760323d#B#compaction#405 average throughput is 2.18 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 57.69 MB/second 2024-11-24T22:16:06,301 INFO [RS:0;bd6592c37bb2:40087-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 744bbec6934be304e01b009be760323d#A#compaction#404 average throughput is 2.18 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 57.69 MB/second 2024-11-24T22:16:06,301 DEBUG [RS:0;bd6592c37bb2:40087-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/744bbec6934be304e01b009be760323d/.tmp/B/39ecf63a40634ca0ac4b613e858c196c is 50, key is test_row_0/B:col10/1732486565041/Put/seqid=0 2024-11-24T22:16:06,301 DEBUG [RS:0;bd6592c37bb2:40087-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/744bbec6934be304e01b009be760323d/.tmp/A/24a77bcdefc14b56820618f3519e5621 is 50, key is test_row_0/A:col10/1732486565041/Put/seqid=0 2024-11-24T22:16:06,305 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45929 is added to blk_1073742308_1484 (size=12493) 2024-11-24T22:16:06,308 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45929 is added to blk_1073742309_1485 (size=12493) 2024-11-24T22:16:06,709 DEBUG [RS:0;bd6592c37bb2:40087-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/744bbec6934be304e01b009be760323d/.tmp/B/39ecf63a40634ca0ac4b613e858c196c as hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/744bbec6934be304e01b009be760323d/B/39ecf63a40634ca0ac4b613e858c196c 2024-11-24T22:16:06,713 DEBUG [RS:0;bd6592c37bb2:40087-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/744bbec6934be304e01b009be760323d/.tmp/A/24a77bcdefc14b56820618f3519e5621 as hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/744bbec6934be304e01b009be760323d/A/24a77bcdefc14b56820618f3519e5621 2024-11-24T22:16:06,714 INFO [RS:0;bd6592c37bb2:40087-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 744bbec6934be304e01b009be760323d/B of 744bbec6934be304e01b009be760323d into 39ecf63a40634ca0ac4b613e858c196c(size=12.2 K), total size for store is 12.2 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-24T22:16:06,714 DEBUG [RS:0;bd6592c37bb2:40087-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 744bbec6934be304e01b009be760323d: 2024-11-24T22:16:06,714 INFO [RS:0;bd6592c37bb2:40087-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732486556329.744bbec6934be304e01b009be760323d., storeName=744bbec6934be304e01b009be760323d/B, priority=12, startTime=1732486566290; duration=0sec 2024-11-24T22:16:06,714 DEBUG [RS:0;bd6592c37bb2:40087-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-24T22:16:06,714 DEBUG [RS:0;bd6592c37bb2:40087-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 744bbec6934be304e01b009be760323d:B 2024-11-24T22:16:06,714 DEBUG [RS:0;bd6592c37bb2:40087-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-24T22:16:06,715 DEBUG [RS:0;bd6592c37bb2:40087-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 48360 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-24T22:16:06,715 DEBUG [RS:0;bd6592c37bb2:40087-longCompactions-0 {}] regionserver.HStore(1541): 744bbec6934be304e01b009be760323d/C is initiating minor compaction (all files) 2024-11-24T22:16:06,715 INFO [RS:0;bd6592c37bb2:40087-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 744bbec6934be304e01b009be760323d/C in TestAcidGuarantees,,1732486556329.744bbec6934be304e01b009be760323d. 2024-11-24T22:16:06,716 INFO [RS:0;bd6592c37bb2:40087-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/744bbec6934be304e01b009be760323d/C/dfd78c5f1e3446ceb9e6877167ddb6c0, hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/744bbec6934be304e01b009be760323d/C/a7b5aae7560f43f3af42b146d01b7d87, hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/744bbec6934be304e01b009be760323d/C/0ad322a0749345fdb793029953abefd7, hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/744bbec6934be304e01b009be760323d/C/e04a440a3f514792b596de994ba37e38] into tmpdir=hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/744bbec6934be304e01b009be760323d/.tmp, totalSize=47.2 K 2024-11-24T22:16:06,716 DEBUG [RS:0;bd6592c37bb2:40087-longCompactions-0 {}] compactions.Compactor(225): Compacting dfd78c5f1e3446ceb9e6877167ddb6c0, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=95, earliestPutTs=1732486561797 2024-11-24T22:16:06,716 DEBUG [RS:0;bd6592c37bb2:40087-longCompactions-0 {}] compactions.Compactor(225): Compacting a7b5aae7560f43f3af42b146d01b7d87, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=115, earliestPutTs=1732486562026 2024-11-24T22:16:06,717 DEBUG [RS:0;bd6592c37bb2:40087-longCompactions-0 {}] compactions.Compactor(225): Compacting 0ad322a0749345fdb793029953abefd7, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=130, earliestPutTs=1732486562926 2024-11-24T22:16:06,717 DEBUG [RS:0;bd6592c37bb2:40087-longCompactions-0 {}] compactions.Compactor(225): Compacting e04a440a3f514792b596de994ba37e38, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=151, earliestPutTs=1732486564371 2024-11-24T22:16:06,717 INFO [RS:0;bd6592c37bb2:40087-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 744bbec6934be304e01b009be760323d/A of 744bbec6934be304e01b009be760323d into 24a77bcdefc14b56820618f3519e5621(size=12.2 K), total size for store is 12.2 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-24T22:16:06,717 DEBUG [RS:0;bd6592c37bb2:40087-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 744bbec6934be304e01b009be760323d: 2024-11-24T22:16:06,717 INFO [RS:0;bd6592c37bb2:40087-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732486556329.744bbec6934be304e01b009be760323d., storeName=744bbec6934be304e01b009be760323d/A, priority=12, startTime=1732486566290; duration=0sec 2024-11-24T22:16:06,718 DEBUG [RS:0;bd6592c37bb2:40087-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-24T22:16:06,718 DEBUG [RS:0;bd6592c37bb2:40087-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 744bbec6934be304e01b009be760323d:A 2024-11-24T22:16:06,725 INFO [RS:0;bd6592c37bb2:40087-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 744bbec6934be304e01b009be760323d#C#compaction#406 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 57.69 MB/second 2024-11-24T22:16:06,725 DEBUG [RS:0;bd6592c37bb2:40087-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/744bbec6934be304e01b009be760323d/.tmp/C/98ae469563954c268cb17c800408d0ea is 50, key is test_row_0/C:col10/1732486565041/Put/seqid=0 2024-11-24T22:16:06,731 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45929 is added to blk_1073742310_1486 (size=12493) 2024-11-24T22:16:06,981 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37701 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=125 2024-11-24T22:16:06,981 INFO [RPCClient-NioEventLoopGroup-4-10 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-24T22:16:06,982 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37701 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-24T22:16:06,983 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37701 {}] procedure2.ProcedureExecutor(1139): Stored pid=127, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=127, table=TestAcidGuarantees 2024-11-24T22:16:06,984 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37701 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=127 2024-11-24T22:16:06,984 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=127, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=127, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-24T22:16:06,985 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=127, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=127, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-24T22:16:06,985 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=128, ppid=127, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-24T22:16:07,091 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37701 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=127 2024-11-24T22:16:07,133 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40087 {}] regionserver.HRegion(8855): Flush requested on 744bbec6934be304e01b009be760323d 2024-11-24T22:16:07,133 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 744bbec6934be304e01b009be760323d 3/3 column families, dataSize=60.38 KB heapSize=180.05 KB 2024-11-24T22:16:07,133 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 744bbec6934be304e01b009be760323d, store=A 2024-11-24T22:16:07,133 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-24T22:16:07,134 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 744bbec6934be304e01b009be760323d, store=B 2024-11-24T22:16:07,134 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-24T22:16:07,134 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 744bbec6934be304e01b009be760323d, store=C 2024-11-24T22:16:07,134 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-24T22:16:07,137 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=40087 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=128 2024-11-24T22:16:07,137 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bd6592c37bb2:0-2 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732486556329.744bbec6934be304e01b009be760323d. 2024-11-24T22:16:07,137 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bd6592c37bb2:0-2 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732486556329.744bbec6934be304e01b009be760323d. as already flushing 2024-11-24T22:16:07,137 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bd6592c37bb2:0-2 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732486556329.744bbec6934be304e01b009be760323d. 2024-11-24T22:16:07,138 ERROR [RS_FLUSH_OPERATIONS-regionserver/bd6592c37bb2:0-2 {event_type=RS_FLUSH_REGIONS, pid=128}] handler.RSProcedureHandler(58): pid=128 java.io.IOException: Unable to complete flush {ENCODED => 744bbec6934be304e01b009be760323d, NAME => 'TestAcidGuarantees,,1732486556329.744bbec6934be304e01b009be760323d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T22:16:07,138 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bd6592c37bb2:0-2 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=128 java.io.IOException: Unable to complete flush {ENCODED => 744bbec6934be304e01b009be760323d, NAME => 'TestAcidGuarantees,,1732486556329.744bbec6934be304e01b009be760323d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T22:16:07,138 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37701 {}] master.HMaster(4177): Remote procedure failed, pid=128 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 744bbec6934be304e01b009be760323d, NAME => 'TestAcidGuarantees,,1732486556329.744bbec6934be304e01b009be760323d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 744bbec6934be304e01b009be760323d, NAME => 'TestAcidGuarantees,,1732486556329.744bbec6934be304e01b009be760323d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T22:16:07,142 DEBUG [RS:0;bd6592c37bb2:40087-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/744bbec6934be304e01b009be760323d/.tmp/C/98ae469563954c268cb17c800408d0ea as hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/744bbec6934be304e01b009be760323d/C/98ae469563954c268cb17c800408d0ea 2024-11-24T22:16:07,147 INFO [RS:0;bd6592c37bb2:40087-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 744bbec6934be304e01b009be760323d/C of 744bbec6934be304e01b009be760323d into 98ae469563954c268cb17c800408d0ea(size=12.2 K), total size for store is 12.2 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-24T22:16:07,148 DEBUG [RS:0;bd6592c37bb2:40087-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 744bbec6934be304e01b009be760323d: 2024-11-24T22:16:07,148 INFO [RS:0;bd6592c37bb2:40087-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732486556329.744bbec6934be304e01b009be760323d., storeName=744bbec6934be304e01b009be760323d/C, priority=12, startTime=1732486566290; duration=0sec 2024-11-24T22:16:07,148 DEBUG [RS:0;bd6592c37bb2:40087-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-24T22:16:07,148 DEBUG [RS:0;bd6592c37bb2:40087-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 744bbec6934be304e01b009be760323d:C 2024-11-24T22:16:07,154 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/744bbec6934be304e01b009be760323d/.tmp/A/3691c3519de8465991e4b9e6f16216df is 50, key is test_row_0/A:col10/1732486567132/Put/seqid=0 2024-11-24T22:16:07,158 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45929 is added to blk_1073742311_1487 (size=14541) 2024-11-24T22:16:07,162 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=165 (bloomFilter=true), to=hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/744bbec6934be304e01b009be760323d/.tmp/A/3691c3519de8465991e4b9e6f16216df 2024-11-24T22:16:07,169 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/744bbec6934be304e01b009be760323d/.tmp/B/4bf0ad08615e4d288f198e363bda6c8f is 50, key is test_row_0/B:col10/1732486567132/Put/seqid=0 2024-11-24T22:16:07,189 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45929 is added to blk_1073742312_1488 (size=12151) 2024-11-24T22:16:07,208 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40087 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=744bbec6934be304e01b009be760323d, server=bd6592c37bb2,40087,1732486423713 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-24T22:16:07,208 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40087 {}] ipc.CallRunner(138): callId: 33 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46824 deadline: 1732486627199, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=744bbec6934be304e01b009be760323d, server=bd6592c37bb2,40087,1732486423713 2024-11-24T22:16:07,209 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732486556329.744bbec6934be304e01b009be760323d., hostname=bd6592c37bb2,40087,1732486423713, seqNum=2 , the old value is region=TestAcidGuarantees,,1732486556329.744bbec6934be304e01b009be760323d., hostname=bd6592c37bb2,40087,1732486423713, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=744bbec6934be304e01b009be760323d, server=bd6592c37bb2,40087,1732486423713 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T22:16:07,209 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732486556329.744bbec6934be304e01b009be760323d., hostname=bd6592c37bb2,40087,1732486423713, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=744bbec6934be304e01b009be760323d, server=bd6592c37bb2,40087,1732486423713 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T22:16:07,209 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732486556329.744bbec6934be304e01b009be760323d., hostname=bd6592c37bb2,40087,1732486423713, seqNum=2 because the exception is null or not the one we care about 2024-11-24T22:16:07,214 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40087 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=744bbec6934be304e01b009be760323d, server=bd6592c37bb2,40087,1732486423713 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-24T22:16:07,215 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40087 {}] ipc.CallRunner(138): callId: 30 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46820 deadline: 1732486627202, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=744bbec6934be304e01b009be760323d, server=bd6592c37bb2,40087,1732486423713 2024-11-24T22:16:07,215 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40087 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=744bbec6934be304e01b009be760323d, server=bd6592c37bb2,40087,1732486423713 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-24T22:16:07,215 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732486556329.744bbec6934be304e01b009be760323d., hostname=bd6592c37bb2,40087,1732486423713, seqNum=2 , the old value is region=TestAcidGuarantees,,1732486556329.744bbec6934be304e01b009be760323d., hostname=bd6592c37bb2,40087,1732486423713, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=744bbec6934be304e01b009be760323d, server=bd6592c37bb2,40087,1732486423713 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T22:16:07,215 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40087 {}] ipc.CallRunner(138): callId: 28 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46816 deadline: 1732486627208, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=744bbec6934be304e01b009be760323d, server=bd6592c37bb2,40087,1732486423713 2024-11-24T22:16:07,215 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732486556329.744bbec6934be304e01b009be760323d., hostname=bd6592c37bb2,40087,1732486423713, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=744bbec6934be304e01b009be760323d, server=bd6592c37bb2,40087,1732486423713 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T22:16:07,215 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732486556329.744bbec6934be304e01b009be760323d., hostname=bd6592c37bb2,40087,1732486423713, seqNum=2 because the exception is null or not the one we care about 2024-11-24T22:16:07,215 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732486556329.744bbec6934be304e01b009be760323d., hostname=bd6592c37bb2,40087,1732486423713, seqNum=2 , the old value is region=TestAcidGuarantees,,1732486556329.744bbec6934be304e01b009be760323d., hostname=bd6592c37bb2,40087,1732486423713, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=744bbec6934be304e01b009be760323d, server=bd6592c37bb2,40087,1732486423713 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T22:16:07,215 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732486556329.744bbec6934be304e01b009be760323d., hostname=bd6592c37bb2,40087,1732486423713, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=744bbec6934be304e01b009be760323d, server=bd6592c37bb2,40087,1732486423713 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T22:16:07,215 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732486556329.744bbec6934be304e01b009be760323d., hostname=bd6592c37bb2,40087,1732486423713, seqNum=2 because the exception is null or not the one we care about 2024-11-24T22:16:07,291 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=40087 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=128 2024-11-24T22:16:07,291 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bd6592c37bb2:0-0 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732486556329.744bbec6934be304e01b009be760323d. 2024-11-24T22:16:07,291 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bd6592c37bb2:0-0 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732486556329.744bbec6934be304e01b009be760323d. as already flushing 2024-11-24T22:16:07,291 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bd6592c37bb2:0-0 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732486556329.744bbec6934be304e01b009be760323d. 2024-11-24T22:16:07,291 ERROR [RS_FLUSH_OPERATIONS-regionserver/bd6592c37bb2:0-0 {event_type=RS_FLUSH_REGIONS, pid=128}] handler.RSProcedureHandler(58): pid=128 java.io.IOException: Unable to complete flush {ENCODED => 744bbec6934be304e01b009be760323d, NAME => 'TestAcidGuarantees,,1732486556329.744bbec6934be304e01b009be760323d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T22:16:07,291 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bd6592c37bb2:0-0 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=128 java.io.IOException: Unable to complete flush {ENCODED => 744bbec6934be304e01b009be760323d, NAME => 'TestAcidGuarantees,,1732486556329.744bbec6934be304e01b009be760323d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T22:16:07,292 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37701 {}] master.HMaster(4177): Remote procedure failed, pid=128 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 744bbec6934be304e01b009be760323d, NAME => 'TestAcidGuarantees,,1732486556329.744bbec6934be304e01b009be760323d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 744bbec6934be304e01b009be760323d, NAME => 'TestAcidGuarantees,,1732486556329.744bbec6934be304e01b009be760323d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T22:16:07,300 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37701 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=127 2024-11-24T22:16:07,300 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40087 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=744bbec6934be304e01b009be760323d, server=bd6592c37bb2,40087,1732486423713 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-24T22:16:07,301 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40087 {}] ipc.CallRunner(138): callId: 109 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46792 deadline: 1732486627291, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=744bbec6934be304e01b009be760323d, server=bd6592c37bb2,40087,1732486423713 2024-11-24T22:16:07,301 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40087 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=744bbec6934be304e01b009be760323d, server=bd6592c37bb2,40087,1732486423713 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-24T22:16:07,301 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40087 {}] ipc.CallRunner(138): callId: 28 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46842 deadline: 1732486627291, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=744bbec6934be304e01b009be760323d, server=bd6592c37bb2,40087,1732486423713 2024-11-24T22:16:07,301 WARN [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRpcRetryingCaller(177): Call to bd6592c37bb2,40087,1732486423713 for 'test_row_0' in 744bbec6934be304e01b009be760323d of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2186 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=744bbec6934be304e01b009be760323d, server=bd6592c37bb2,40087,1732486423713 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor44.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=744bbec6934be304e01b009be760323d, server=bd6592c37bb2,40087,1732486423713 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-24T22:16:07,301 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732486556329.744bbec6934be304e01b009be760323d., hostname=bd6592c37bb2,40087,1732486423713, seqNum=2 , the old value is region=TestAcidGuarantees,,1732486556329.744bbec6934be304e01b009be760323d., hostname=bd6592c37bb2,40087,1732486423713, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=744bbec6934be304e01b009be760323d, server=bd6592c37bb2,40087,1732486423713 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T22:16:07,301 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732486556329.744bbec6934be304e01b009be760323d., hostname=bd6592c37bb2,40087,1732486423713, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=744bbec6934be304e01b009be760323d, server=bd6592c37bb2,40087,1732486423713 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T22:16:07,302 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732486556329.744bbec6934be304e01b009be760323d., hostname=bd6592c37bb2,40087,1732486423713, seqNum=2 because the exception is null or not the one we care about 2024-11-24T22:16:07,301 WARN [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRpcRetryingCaller(177): Call to bd6592c37bb2,40087,1732486423713 for 'test_row_1' in 744bbec6934be304e01b009be760323d of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2182 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=744bbec6934be304e01b009be760323d, server=bd6592c37bb2,40087,1732486423713 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor44.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=744bbec6934be304e01b009be760323d, server=bd6592c37bb2,40087,1732486423713 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-24T22:16:07,302 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732486556329.744bbec6934be304e01b009be760323d., hostname=bd6592c37bb2,40087,1732486423713, seqNum=2 , the old value is region=TestAcidGuarantees,,1732486556329.744bbec6934be304e01b009be760323d., hostname=bd6592c37bb2,40087,1732486423713, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=744bbec6934be304e01b009be760323d, server=bd6592c37bb2,40087,1732486423713 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T22:16:07,302 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732486556329.744bbec6934be304e01b009be760323d., hostname=bd6592c37bb2,40087,1732486423713, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=744bbec6934be304e01b009be760323d, server=bd6592c37bb2,40087,1732486423713 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T22:16:07,302 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732486556329.744bbec6934be304e01b009be760323d., hostname=bd6592c37bb2,40087,1732486423713, seqNum=2 because the exception is null or not the one we care about 2024-11-24T22:16:07,314 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40087 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=744bbec6934be304e01b009be760323d, server=bd6592c37bb2,40087,1732486423713 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-24T22:16:07,314 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40087 {}] ipc.CallRunner(138): callId: 34 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46824 deadline: 1732486627311, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=744bbec6934be304e01b009be760323d, server=bd6592c37bb2,40087,1732486423713 2024-11-24T22:16:07,314 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732486556329.744bbec6934be304e01b009be760323d., hostname=bd6592c37bb2,40087,1732486423713, seqNum=2 , the old value is region=TestAcidGuarantees,,1732486556329.744bbec6934be304e01b009be760323d., hostname=bd6592c37bb2,40087,1732486423713, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=744bbec6934be304e01b009be760323d, server=bd6592c37bb2,40087,1732486423713 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T22:16:07,314 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732486556329.744bbec6934be304e01b009be760323d., hostname=bd6592c37bb2,40087,1732486423713, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=744bbec6934be304e01b009be760323d, server=bd6592c37bb2,40087,1732486423713 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T22:16:07,314 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732486556329.744bbec6934be304e01b009be760323d., hostname=bd6592c37bb2,40087,1732486423713, seqNum=2 because the exception is null or not the one we care about 2024-11-24T22:16:07,327 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40087 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=744bbec6934be304e01b009be760323d, server=bd6592c37bb2,40087,1732486423713 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-24T22:16:07,327 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40087 {}] ipc.CallRunner(138): callId: 31 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46820 deadline: 1732486627321, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=744bbec6934be304e01b009be760323d, server=bd6592c37bb2,40087,1732486423713 2024-11-24T22:16:07,328 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40087 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=744bbec6934be304e01b009be760323d, server=bd6592c37bb2,40087,1732486423713 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-24T22:16:07,328 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732486556329.744bbec6934be304e01b009be760323d., hostname=bd6592c37bb2,40087,1732486423713, seqNum=2 , the old value is region=TestAcidGuarantees,,1732486556329.744bbec6934be304e01b009be760323d., hostname=bd6592c37bb2,40087,1732486423713, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=744bbec6934be304e01b009be760323d, server=bd6592c37bb2,40087,1732486423713 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T22:16:07,328 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732486556329.744bbec6934be304e01b009be760323d., hostname=bd6592c37bb2,40087,1732486423713, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=744bbec6934be304e01b009be760323d, server=bd6592c37bb2,40087,1732486423713 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T22:16:07,328 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732486556329.744bbec6934be304e01b009be760323d., hostname=bd6592c37bb2,40087,1732486423713, seqNum=2 because the exception is null or not the one we care about 2024-11-24T22:16:07,328 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40087 {}] ipc.CallRunner(138): callId: 29 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46816 deadline: 1732486627321, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=744bbec6934be304e01b009be760323d, server=bd6592c37bb2,40087,1732486423713 2024-11-24T22:16:07,328 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732486556329.744bbec6934be304e01b009be760323d., hostname=bd6592c37bb2,40087,1732486423713, seqNum=2 , the old value is region=TestAcidGuarantees,,1732486556329.744bbec6934be304e01b009be760323d., hostname=bd6592c37bb2,40087,1732486423713, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=744bbec6934be304e01b009be760323d, server=bd6592c37bb2,40087,1732486423713 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T22:16:07,328 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732486556329.744bbec6934be304e01b009be760323d., hostname=bd6592c37bb2,40087,1732486423713, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=744bbec6934be304e01b009be760323d, server=bd6592c37bb2,40087,1732486423713 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T22:16:07,328 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732486556329.744bbec6934be304e01b009be760323d., hostname=bd6592c37bb2,40087,1732486423713, seqNum=2 because the exception is null or not the one we care about 2024-11-24T22:16:07,444 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=40087 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=128 2024-11-24T22:16:07,444 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bd6592c37bb2:0-1 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732486556329.744bbec6934be304e01b009be760323d. 2024-11-24T22:16:07,444 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bd6592c37bb2:0-1 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732486556329.744bbec6934be304e01b009be760323d. as already flushing 2024-11-24T22:16:07,444 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bd6592c37bb2:0-1 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732486556329.744bbec6934be304e01b009be760323d. 2024-11-24T22:16:07,444 ERROR [RS_FLUSH_OPERATIONS-regionserver/bd6592c37bb2:0-1 {event_type=RS_FLUSH_REGIONS, pid=128}] handler.RSProcedureHandler(58): pid=128 java.io.IOException: Unable to complete flush {ENCODED => 744bbec6934be304e01b009be760323d, NAME => 'TestAcidGuarantees,,1732486556329.744bbec6934be304e01b009be760323d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T22:16:07,444 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bd6592c37bb2:0-1 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=128 java.io.IOException: Unable to complete flush {ENCODED => 744bbec6934be304e01b009be760323d, NAME => 'TestAcidGuarantees,,1732486556329.744bbec6934be304e01b009be760323d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T22:16:07,445 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37701 {}] master.HMaster(4177): Remote procedure failed, pid=128 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 744bbec6934be304e01b009be760323d, NAME => 'TestAcidGuarantees,,1732486556329.744bbec6934be304e01b009be760323d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 744bbec6934be304e01b009be760323d, NAME => 'TestAcidGuarantees,,1732486556329.744bbec6934be304e01b009be760323d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T22:16:07,530 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40087 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=744bbec6934be304e01b009be760323d, server=bd6592c37bb2,40087,1732486423713 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-24T22:16:07,530 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40087 {}] ipc.CallRunner(138): callId: 35 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46824 deadline: 1732486627520, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=744bbec6934be304e01b009be760323d, server=bd6592c37bb2,40087,1732486423713 2024-11-24T22:16:07,530 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732486556329.744bbec6934be304e01b009be760323d., hostname=bd6592c37bb2,40087,1732486423713, seqNum=2 , the old value is region=TestAcidGuarantees,,1732486556329.744bbec6934be304e01b009be760323d., hostname=bd6592c37bb2,40087,1732486423713, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=744bbec6934be304e01b009be760323d, server=bd6592c37bb2,40087,1732486423713 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T22:16:07,530 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732486556329.744bbec6934be304e01b009be760323d., hostname=bd6592c37bb2,40087,1732486423713, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=744bbec6934be304e01b009be760323d, server=bd6592c37bb2,40087,1732486423713 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T22:16:07,530 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732486556329.744bbec6934be304e01b009be760323d., hostname=bd6592c37bb2,40087,1732486423713, seqNum=2 because the exception is null or not the one we care about 2024-11-24T22:16:07,541 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40087 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=744bbec6934be304e01b009be760323d, server=bd6592c37bb2,40087,1732486423713 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-24T22:16:07,541 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40087 {}] ipc.CallRunner(138): callId: 32 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46820 deadline: 1732486627531, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=744bbec6934be304e01b009be760323d, server=bd6592c37bb2,40087,1732486423713 2024-11-24T22:16:07,541 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40087 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=744bbec6934be304e01b009be760323d, server=bd6592c37bb2,40087,1732486423713 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-24T22:16:07,541 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732486556329.744bbec6934be304e01b009be760323d., hostname=bd6592c37bb2,40087,1732486423713, seqNum=2 , the old value is region=TestAcidGuarantees,,1732486556329.744bbec6934be304e01b009be760323d., hostname=bd6592c37bb2,40087,1732486423713, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=744bbec6934be304e01b009be760323d, server=bd6592c37bb2,40087,1732486423713 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T22:16:07,542 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40087 {}] ipc.CallRunner(138): callId: 30 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46816 deadline: 1732486627531, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=744bbec6934be304e01b009be760323d, server=bd6592c37bb2,40087,1732486423713 2024-11-24T22:16:07,542 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732486556329.744bbec6934be304e01b009be760323d., hostname=bd6592c37bb2,40087,1732486423713, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=744bbec6934be304e01b009be760323d, server=bd6592c37bb2,40087,1732486423713 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T22:16:07,542 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732486556329.744bbec6934be304e01b009be760323d., hostname=bd6592c37bb2,40087,1732486423713, seqNum=2 because the exception is null or not the one we care about 2024-11-24T22:16:07,542 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732486556329.744bbec6934be304e01b009be760323d., hostname=bd6592c37bb2,40087,1732486423713, seqNum=2 , the old value is region=TestAcidGuarantees,,1732486556329.744bbec6934be304e01b009be760323d., hostname=bd6592c37bb2,40087,1732486423713, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=744bbec6934be304e01b009be760323d, server=bd6592c37bb2,40087,1732486423713 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T22:16:07,542 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732486556329.744bbec6934be304e01b009be760323d., hostname=bd6592c37bb2,40087,1732486423713, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=744bbec6934be304e01b009be760323d, server=bd6592c37bb2,40087,1732486423713 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T22:16:07,542 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732486556329.744bbec6934be304e01b009be760323d., hostname=bd6592c37bb2,40087,1732486423713, seqNum=2 because the exception is null or not the one we care about 2024-11-24T22:16:07,590 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=165 (bloomFilter=true), to=hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/744bbec6934be304e01b009be760323d/.tmp/B/4bf0ad08615e4d288f198e363bda6c8f 2024-11-24T22:16:07,596 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=40087 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=128 2024-11-24T22:16:07,596 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bd6592c37bb2:0-2 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732486556329.744bbec6934be304e01b009be760323d. 2024-11-24T22:16:07,597 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bd6592c37bb2:0-2 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732486556329.744bbec6934be304e01b009be760323d. as already flushing 2024-11-24T22:16:07,597 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/744bbec6934be304e01b009be760323d/.tmp/C/958ad208c5ba4d749d3f1c406bdcd7f7 is 50, key is test_row_0/C:col10/1732486567132/Put/seqid=0 2024-11-24T22:16:07,597 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bd6592c37bb2:0-2 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732486556329.744bbec6934be304e01b009be760323d. 2024-11-24T22:16:07,597 ERROR [RS_FLUSH_OPERATIONS-regionserver/bd6592c37bb2:0-2 {event_type=RS_FLUSH_REGIONS, pid=128}] handler.RSProcedureHandler(58): pid=128 java.io.IOException: Unable to complete flush {ENCODED => 744bbec6934be304e01b009be760323d, NAME => 'TestAcidGuarantees,,1732486556329.744bbec6934be304e01b009be760323d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T22:16:07,597 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bd6592c37bb2:0-2 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=128 java.io.IOException: Unable to complete flush {ENCODED => 744bbec6934be304e01b009be760323d, NAME => 'TestAcidGuarantees,,1732486556329.744bbec6934be304e01b009be760323d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T22:16:07,597 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37701 {}] master.HMaster(4177): Remote procedure failed, pid=128 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 744bbec6934be304e01b009be760323d, NAME => 'TestAcidGuarantees,,1732486556329.744bbec6934be304e01b009be760323d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 744bbec6934be304e01b009be760323d, NAME => 'TestAcidGuarantees,,1732486556329.744bbec6934be304e01b009be760323d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T22:16:07,600 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45929 is added to blk_1073742313_1489 (size=12151) 2024-11-24T22:16:07,610 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37701 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=127 2024-11-24T22:16:07,749 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=40087 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=128 2024-11-24T22:16:07,749 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bd6592c37bb2:0-0 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732486556329.744bbec6934be304e01b009be760323d. 2024-11-24T22:16:07,749 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bd6592c37bb2:0-0 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732486556329.744bbec6934be304e01b009be760323d. as already flushing 2024-11-24T22:16:07,750 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bd6592c37bb2:0-0 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732486556329.744bbec6934be304e01b009be760323d. 2024-11-24T22:16:07,750 ERROR [RS_FLUSH_OPERATIONS-regionserver/bd6592c37bb2:0-0 {event_type=RS_FLUSH_REGIONS, pid=128}] handler.RSProcedureHandler(58): pid=128 java.io.IOException: Unable to complete flush {ENCODED => 744bbec6934be304e01b009be760323d, NAME => 'TestAcidGuarantees,,1732486556329.744bbec6934be304e01b009be760323d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T22:16:07,750 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bd6592c37bb2:0-0 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=128 java.io.IOException: Unable to complete flush {ENCODED => 744bbec6934be304e01b009be760323d, NAME => 'TestAcidGuarantees,,1732486556329.744bbec6934be304e01b009be760323d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T22:16:07,750 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37701 {}] master.HMaster(4177): Remote procedure failed, pid=128 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 744bbec6934be304e01b009be760323d, NAME => 'TestAcidGuarantees,,1732486556329.744bbec6934be304e01b009be760323d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 744bbec6934be304e01b009be760323d, NAME => 'TestAcidGuarantees,,1732486556329.744bbec6934be304e01b009be760323d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T22:16:07,851 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40087 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=744bbec6934be304e01b009be760323d, server=bd6592c37bb2,40087,1732486423713 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-24T22:16:07,851 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40087 {}] ipc.CallRunner(138): callId: 36 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46824 deadline: 1732486627840, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=744bbec6934be304e01b009be760323d, server=bd6592c37bb2,40087,1732486423713 2024-11-24T22:16:07,852 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732486556329.744bbec6934be304e01b009be760323d., hostname=bd6592c37bb2,40087,1732486423713, seqNum=2 , the old value is region=TestAcidGuarantees,,1732486556329.744bbec6934be304e01b009be760323d., hostname=bd6592c37bb2,40087,1732486423713, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=744bbec6934be304e01b009be760323d, server=bd6592c37bb2,40087,1732486423713 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T22:16:07,852 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732486556329.744bbec6934be304e01b009be760323d., hostname=bd6592c37bb2,40087,1732486423713, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=744bbec6934be304e01b009be760323d, server=bd6592c37bb2,40087,1732486423713 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T22:16:07,852 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732486556329.744bbec6934be304e01b009be760323d., hostname=bd6592c37bb2,40087,1732486423713, seqNum=2 because the exception is null or not the one we care about 2024-11-24T22:16:07,860 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40087 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=744bbec6934be304e01b009be760323d, server=bd6592c37bb2,40087,1732486423713 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-24T22:16:07,860 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40087 {}] ipc.CallRunner(138): callId: 33 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46820 deadline: 1732486627850, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=744bbec6934be304e01b009be760323d, server=bd6592c37bb2,40087,1732486423713 2024-11-24T22:16:07,860 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40087 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=744bbec6934be304e01b009be760323d, server=bd6592c37bb2,40087,1732486423713 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-24T22:16:07,860 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732486556329.744bbec6934be304e01b009be760323d., hostname=bd6592c37bb2,40087,1732486423713, seqNum=2 , the old value is region=TestAcidGuarantees,,1732486556329.744bbec6934be304e01b009be760323d., hostname=bd6592c37bb2,40087,1732486423713, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=744bbec6934be304e01b009be760323d, server=bd6592c37bb2,40087,1732486423713 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T22:16:07,860 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40087 {}] ipc.CallRunner(138): callId: 31 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46816 deadline: 1732486627851, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=744bbec6934be304e01b009be760323d, server=bd6592c37bb2,40087,1732486423713 2024-11-24T22:16:07,860 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732486556329.744bbec6934be304e01b009be760323d., hostname=bd6592c37bb2,40087,1732486423713, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=744bbec6934be304e01b009be760323d, server=bd6592c37bb2,40087,1732486423713 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T22:16:07,860 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732486556329.744bbec6934be304e01b009be760323d., hostname=bd6592c37bb2,40087,1732486423713, seqNum=2 because the exception is null or not the one we care about 2024-11-24T22:16:07,860 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732486556329.744bbec6934be304e01b009be760323d., hostname=bd6592c37bb2,40087,1732486423713, seqNum=2 , the old value is region=TestAcidGuarantees,,1732486556329.744bbec6934be304e01b009be760323d., hostname=bd6592c37bb2,40087,1732486423713, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=744bbec6934be304e01b009be760323d, server=bd6592c37bb2,40087,1732486423713 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T22:16:07,860 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732486556329.744bbec6934be304e01b009be760323d., hostname=bd6592c37bb2,40087,1732486423713, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=744bbec6934be304e01b009be760323d, server=bd6592c37bb2,40087,1732486423713 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T22:16:07,861 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732486556329.744bbec6934be304e01b009be760323d., hostname=bd6592c37bb2,40087,1732486423713, seqNum=2 because the exception is null or not the one we care about 2024-11-24T22:16:07,902 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=40087 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=128 2024-11-24T22:16:07,902 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bd6592c37bb2:0-1 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732486556329.744bbec6934be304e01b009be760323d. 2024-11-24T22:16:07,902 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bd6592c37bb2:0-1 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732486556329.744bbec6934be304e01b009be760323d. as already flushing 2024-11-24T22:16:07,902 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bd6592c37bb2:0-1 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732486556329.744bbec6934be304e01b009be760323d. 2024-11-24T22:16:07,903 ERROR [RS_FLUSH_OPERATIONS-regionserver/bd6592c37bb2:0-1 {event_type=RS_FLUSH_REGIONS, pid=128}] handler.RSProcedureHandler(58): pid=128 java.io.IOException: Unable to complete flush {ENCODED => 744bbec6934be304e01b009be760323d, NAME => 'TestAcidGuarantees,,1732486556329.744bbec6934be304e01b009be760323d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T22:16:07,903 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bd6592c37bb2:0-1 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=128 java.io.IOException: Unable to complete flush {ENCODED => 744bbec6934be304e01b009be760323d, NAME => 'TestAcidGuarantees,,1732486556329.744bbec6934be304e01b009be760323d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T22:16:07,903 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37701 {}] master.HMaster(4177): Remote procedure failed, pid=128 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 744bbec6934be304e01b009be760323d, NAME => 'TestAcidGuarantees,,1732486556329.744bbec6934be304e01b009be760323d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 744bbec6934be304e01b009be760323d, NAME => 'TestAcidGuarantees,,1732486556329.744bbec6934be304e01b009be760323d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T22:16:08,000 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=165 (bloomFilter=true), to=hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/744bbec6934be304e01b009be760323d/.tmp/C/958ad208c5ba4d749d3f1c406bdcd7f7 2024-11-24T22:16:08,004 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/744bbec6934be304e01b009be760323d/.tmp/A/3691c3519de8465991e4b9e6f16216df as hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/744bbec6934be304e01b009be760323d/A/3691c3519de8465991e4b9e6f16216df 2024-11-24T22:16:08,007 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/744bbec6934be304e01b009be760323d/A/3691c3519de8465991e4b9e6f16216df, entries=200, sequenceid=165, filesize=14.2 K 2024-11-24T22:16:08,008 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/744bbec6934be304e01b009be760323d/.tmp/B/4bf0ad08615e4d288f198e363bda6c8f as hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/744bbec6934be304e01b009be760323d/B/4bf0ad08615e4d288f198e363bda6c8f 2024-11-24T22:16:08,010 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/744bbec6934be304e01b009be760323d/B/4bf0ad08615e4d288f198e363bda6c8f, entries=150, sequenceid=165, filesize=11.9 K 2024-11-24T22:16:08,011 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/744bbec6934be304e01b009be760323d/.tmp/C/958ad208c5ba4d749d3f1c406bdcd7f7 as hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/744bbec6934be304e01b009be760323d/C/958ad208c5ba4d749d3f1c406bdcd7f7 2024-11-24T22:16:08,014 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/744bbec6934be304e01b009be760323d/C/958ad208c5ba4d749d3f1c406bdcd7f7, entries=150, sequenceid=165, filesize=11.9 K 2024-11-24T22:16:08,015 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~60.38 KB/61830, heapSize ~180 KB/184320, currentSize=114.05 KB/116790 for 744bbec6934be304e01b009be760323d in 882ms, sequenceid=165, compaction requested=false 2024-11-24T22:16:08,015 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 744bbec6934be304e01b009be760323d: 2024-11-24T22:16:08,055 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=40087 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=128 2024-11-24T22:16:08,055 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bd6592c37bb2:0-2 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732486556329.744bbec6934be304e01b009be760323d. 2024-11-24T22:16:08,056 INFO [RS_FLUSH_OPERATIONS-regionserver/bd6592c37bb2:0-2 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.HRegion(2902): Flushing 744bbec6934be304e01b009be760323d 3/3 column families, dataSize=114.05 KB heapSize=339.42 KB 2024-11-24T22:16:08,056 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bd6592c37bb2:0-2 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 744bbec6934be304e01b009be760323d, store=A 2024-11-24T22:16:08,056 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bd6592c37bb2:0-2 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-24T22:16:08,056 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bd6592c37bb2:0-2 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 744bbec6934be304e01b009be760323d, store=B 2024-11-24T22:16:08,056 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bd6592c37bb2:0-2 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-24T22:16:08,056 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bd6592c37bb2:0-2 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 744bbec6934be304e01b009be760323d, store=C 2024-11-24T22:16:08,056 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bd6592c37bb2:0-2 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-24T22:16:08,060 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bd6592c37bb2:0-2 {event_type=RS_FLUSH_REGIONS, pid=128}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/744bbec6934be304e01b009be760323d/.tmp/A/f98aceb87462436897276c8ce48119a3 is 50, key is test_row_0/A:col10/1732486567160/Put/seqid=0 2024-11-24T22:16:08,064 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45929 is added to blk_1073742314_1490 (size=12151) 2024-11-24T22:16:08,121 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37701 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=127 2024-11-24T22:16:08,367 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40087 {}] regionserver.HRegion(8855): Flush requested on 744bbec6934be304e01b009be760323d 2024-11-24T22:16:08,367 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732486556329.744bbec6934be304e01b009be760323d. as already flushing 2024-11-24T22:16:08,400 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40087 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=744bbec6934be304e01b009be760323d, server=bd6592c37bb2,40087,1732486423713 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-24T22:16:08,400 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40087 {}] ipc.CallRunner(138): callId: 34 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46816 deadline: 1732486628394, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=744bbec6934be304e01b009be760323d, server=bd6592c37bb2,40087,1732486423713 2024-11-24T22:16:08,400 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732486556329.744bbec6934be304e01b009be760323d., hostname=bd6592c37bb2,40087,1732486423713, seqNum=2 , the old value is region=TestAcidGuarantees,,1732486556329.744bbec6934be304e01b009be760323d., hostname=bd6592c37bb2,40087,1732486423713, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=744bbec6934be304e01b009be760323d, server=bd6592c37bb2,40087,1732486423713 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T22:16:08,400 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732486556329.744bbec6934be304e01b009be760323d., hostname=bd6592c37bb2,40087,1732486423713, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=744bbec6934be304e01b009be760323d, server=bd6592c37bb2,40087,1732486423713 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T22:16:08,400 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40087 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=744bbec6934be304e01b009be760323d, server=bd6592c37bb2,40087,1732486423713 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-24T22:16:08,400 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732486556329.744bbec6934be304e01b009be760323d., hostname=bd6592c37bb2,40087,1732486423713, seqNum=2 because the exception is null or not the one we care about 2024-11-24T22:16:08,400 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40087 {}] ipc.CallRunner(138): callId: 41 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46824 deadline: 1732486628395, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=744bbec6934be304e01b009be760323d, server=bd6592c37bb2,40087,1732486423713 2024-11-24T22:16:08,400 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732486556329.744bbec6934be304e01b009be760323d., hostname=bd6592c37bb2,40087,1732486423713, seqNum=2 , the old value is region=TestAcidGuarantees,,1732486556329.744bbec6934be304e01b009be760323d., hostname=bd6592c37bb2,40087,1732486423713, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=744bbec6934be304e01b009be760323d, server=bd6592c37bb2,40087,1732486423713 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T22:16:08,401 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732486556329.744bbec6934be304e01b009be760323d., hostname=bd6592c37bb2,40087,1732486423713, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=744bbec6934be304e01b009be760323d, server=bd6592c37bb2,40087,1732486423713 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T22:16:08,401 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732486556329.744bbec6934be304e01b009be760323d., hostname=bd6592c37bb2,40087,1732486423713, seqNum=2 because the exception is null or not the one we care about 2024-11-24T22:16:08,405 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40087 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=744bbec6934be304e01b009be760323d, server=bd6592c37bb2,40087,1732486423713 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-24T22:16:08,406 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40087 {}] ipc.CallRunner(138): callId: 37 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46820 deadline: 1732486628396, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=744bbec6934be304e01b009be760323d, server=bd6592c37bb2,40087,1732486423713 2024-11-24T22:16:08,406 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732486556329.744bbec6934be304e01b009be760323d., hostname=bd6592c37bb2,40087,1732486423713, seqNum=2 , the old value is region=TestAcidGuarantees,,1732486556329.744bbec6934be304e01b009be760323d., hostname=bd6592c37bb2,40087,1732486423713, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=744bbec6934be304e01b009be760323d, server=bd6592c37bb2,40087,1732486423713 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T22:16:08,406 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732486556329.744bbec6934be304e01b009be760323d., hostname=bd6592c37bb2,40087,1732486423713, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=744bbec6934be304e01b009be760323d, server=bd6592c37bb2,40087,1732486423713 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T22:16:08,406 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732486556329.744bbec6934be304e01b009be760323d., hostname=bd6592c37bb2,40087,1732486423713, seqNum=2 because the exception is null or not the one we care about 2024-11-24T22:16:08,465 INFO [RS_FLUSH_OPERATIONS-regionserver/bd6592c37bb2:0-2 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=38.02 KB at sequenceid=186 (bloomFilter=true), to=hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/744bbec6934be304e01b009be760323d/.tmp/A/f98aceb87462436897276c8ce48119a3 2024-11-24T22:16:08,471 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bd6592c37bb2:0-2 {event_type=RS_FLUSH_REGIONS, pid=128}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/744bbec6934be304e01b009be760323d/.tmp/B/858191303eae4384ae680814f2697353 is 50, key is test_row_0/B:col10/1732486567160/Put/seqid=0 2024-11-24T22:16:08,474 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45929 is added to blk_1073742315_1491 (size=12151) 2024-11-24T22:16:08,521 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40087 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=744bbec6934be304e01b009be760323d, server=bd6592c37bb2,40087,1732486423713 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-24T22:16:08,521 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40087 {}] ipc.CallRunner(138): callId: 35 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46816 deadline: 1732486628511, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=744bbec6934be304e01b009be760323d, server=bd6592c37bb2,40087,1732486423713 2024-11-24T22:16:08,521 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40087 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=744bbec6934be304e01b009be760323d, server=bd6592c37bb2,40087,1732486423713 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-24T22:16:08,521 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732486556329.744bbec6934be304e01b009be760323d., hostname=bd6592c37bb2,40087,1732486423713, seqNum=2 , the old value is region=TestAcidGuarantees,,1732486556329.744bbec6934be304e01b009be760323d., hostname=bd6592c37bb2,40087,1732486423713, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=744bbec6934be304e01b009be760323d, server=bd6592c37bb2,40087,1732486423713 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T22:16:08,521 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732486556329.744bbec6934be304e01b009be760323d., hostname=bd6592c37bb2,40087,1732486423713, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=744bbec6934be304e01b009be760323d, server=bd6592c37bb2,40087,1732486423713 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T22:16:08,521 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40087 {}] ipc.CallRunner(138): callId: 42 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46824 deadline: 1732486628511, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=744bbec6934be304e01b009be760323d, server=bd6592c37bb2,40087,1732486423713 2024-11-24T22:16:08,521 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732486556329.744bbec6934be304e01b009be760323d., hostname=bd6592c37bb2,40087,1732486423713, seqNum=2 because the exception is null or not the one we care about 2024-11-24T22:16:08,521 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40087 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=744bbec6934be304e01b009be760323d, server=bd6592c37bb2,40087,1732486423713 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-24T22:16:08,521 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732486556329.744bbec6934be304e01b009be760323d., hostname=bd6592c37bb2,40087,1732486423713, seqNum=2 , the old value is region=TestAcidGuarantees,,1732486556329.744bbec6934be304e01b009be760323d., hostname=bd6592c37bb2,40087,1732486423713, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=744bbec6934be304e01b009be760323d, server=bd6592c37bb2,40087,1732486423713 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T22:16:08,521 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732486556329.744bbec6934be304e01b009be760323d., hostname=bd6592c37bb2,40087,1732486423713, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=744bbec6934be304e01b009be760323d, server=bd6592c37bb2,40087,1732486423713 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T22:16:08,521 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40087 {}] ipc.CallRunner(138): callId: 38 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46820 deadline: 1732486628511, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=744bbec6934be304e01b009be760323d, server=bd6592c37bb2,40087,1732486423713 2024-11-24T22:16:08,521 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732486556329.744bbec6934be304e01b009be760323d., hostname=bd6592c37bb2,40087,1732486423713, seqNum=2 because the exception is null or not the one we care about 2024-11-24T22:16:08,522 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732486556329.744bbec6934be304e01b009be760323d., hostname=bd6592c37bb2,40087,1732486423713, seqNum=2 , the old value is region=TestAcidGuarantees,,1732486556329.744bbec6934be304e01b009be760323d., hostname=bd6592c37bb2,40087,1732486423713, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=744bbec6934be304e01b009be760323d, server=bd6592c37bb2,40087,1732486423713 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T22:16:08,522 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732486556329.744bbec6934be304e01b009be760323d., hostname=bd6592c37bb2,40087,1732486423713, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=744bbec6934be304e01b009be760323d, server=bd6592c37bb2,40087,1732486423713 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T22:16:08,522 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732486556329.744bbec6934be304e01b009be760323d., hostname=bd6592c37bb2,40087,1732486423713, seqNum=2 because the exception is null or not the one we care about 2024-11-24T22:16:08,741 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40087 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=744bbec6934be304e01b009be760323d, server=bd6592c37bb2,40087,1732486423713 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-24T22:16:08,741 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40087 {}] ipc.CallRunner(138): callId: 36 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46816 deadline: 1732486628731, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=744bbec6934be304e01b009be760323d, server=bd6592c37bb2,40087,1732486423713 2024-11-24T22:16:08,741 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40087 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=744bbec6934be304e01b009be760323d, server=bd6592c37bb2,40087,1732486423713 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-24T22:16:08,742 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732486556329.744bbec6934be304e01b009be760323d., hostname=bd6592c37bb2,40087,1732486423713, seqNum=2 , the old value is region=TestAcidGuarantees,,1732486556329.744bbec6934be304e01b009be760323d., hostname=bd6592c37bb2,40087,1732486423713, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=744bbec6934be304e01b009be760323d, server=bd6592c37bb2,40087,1732486423713 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T22:16:08,742 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40087 {}] ipc.CallRunner(138): callId: 43 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46824 deadline: 1732486628731, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=744bbec6934be304e01b009be760323d, server=bd6592c37bb2,40087,1732486423713 2024-11-24T22:16:08,742 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732486556329.744bbec6934be304e01b009be760323d., hostname=bd6592c37bb2,40087,1732486423713, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=744bbec6934be304e01b009be760323d, server=bd6592c37bb2,40087,1732486423713 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T22:16:08,742 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732486556329.744bbec6934be304e01b009be760323d., hostname=bd6592c37bb2,40087,1732486423713, seqNum=2 because the exception is null or not the one we care about 2024-11-24T22:16:08,742 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40087 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=744bbec6934be304e01b009be760323d, server=bd6592c37bb2,40087,1732486423713 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-24T22:16:08,742 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40087 {}] ipc.CallRunner(138): callId: 39 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46820 deadline: 1732486628731, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=744bbec6934be304e01b009be760323d, server=bd6592c37bb2,40087,1732486423713 2024-11-24T22:16:08,742 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732486556329.744bbec6934be304e01b009be760323d., hostname=bd6592c37bb2,40087,1732486423713, seqNum=2 , the old value is region=TestAcidGuarantees,,1732486556329.744bbec6934be304e01b009be760323d., hostname=bd6592c37bb2,40087,1732486423713, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=744bbec6934be304e01b009be760323d, server=bd6592c37bb2,40087,1732486423713 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T22:16:08,742 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732486556329.744bbec6934be304e01b009be760323d., hostname=bd6592c37bb2,40087,1732486423713, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=744bbec6934be304e01b009be760323d, server=bd6592c37bb2,40087,1732486423713 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T22:16:08,742 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732486556329.744bbec6934be304e01b009be760323d., hostname=bd6592c37bb2,40087,1732486423713, seqNum=2 because the exception is null or not the one we care about 2024-11-24T22:16:08,742 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732486556329.744bbec6934be304e01b009be760323d., hostname=bd6592c37bb2,40087,1732486423713, seqNum=2 , the old value is region=TestAcidGuarantees,,1732486556329.744bbec6934be304e01b009be760323d., hostname=bd6592c37bb2,40087,1732486423713, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=744bbec6934be304e01b009be760323d, server=bd6592c37bb2,40087,1732486423713 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T22:16:08,742 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732486556329.744bbec6934be304e01b009be760323d., hostname=bd6592c37bb2,40087,1732486423713, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=744bbec6934be304e01b009be760323d, server=bd6592c37bb2,40087,1732486423713 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T22:16:08,742 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732486556329.744bbec6934be304e01b009be760323d., hostname=bd6592c37bb2,40087,1732486423713, seqNum=2 because the exception is null or not the one we care about 2024-11-24T22:16:08,875 INFO [RS_FLUSH_OPERATIONS-regionserver/bd6592c37bb2:0-2 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=38.02 KB at sequenceid=186 (bloomFilter=true), to=hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/744bbec6934be304e01b009be760323d/.tmp/B/858191303eae4384ae680814f2697353 2024-11-24T22:16:08,882 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bd6592c37bb2:0-2 {event_type=RS_FLUSH_REGIONS, pid=128}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/744bbec6934be304e01b009be760323d/.tmp/C/d53b3ca00fc140c6b59fdb5c520720c7 is 50, key is test_row_0/C:col10/1732486567160/Put/seqid=0 2024-11-24T22:16:08,885 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45929 is added to blk_1073742316_1492 (size=12151) 2024-11-24T22:16:09,058 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40087 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=744bbec6934be304e01b009be760323d, server=bd6592c37bb2,40087,1732486423713 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-24T22:16:09,058 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40087 {}] ipc.CallRunner(138): callId: 37 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46816 deadline: 1732486629051, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=744bbec6934be304e01b009be760323d, server=bd6592c37bb2,40087,1732486423713 2024-11-24T22:16:09,058 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732486556329.744bbec6934be304e01b009be760323d., hostname=bd6592c37bb2,40087,1732486423713, seqNum=2 , the old value is region=TestAcidGuarantees,,1732486556329.744bbec6934be304e01b009be760323d., hostname=bd6592c37bb2,40087,1732486423713, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=744bbec6934be304e01b009be760323d, server=bd6592c37bb2,40087,1732486423713 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T22:16:09,058 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40087 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=744bbec6934be304e01b009be760323d, server=bd6592c37bb2,40087,1732486423713 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-24T22:16:09,058 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732486556329.744bbec6934be304e01b009be760323d., hostname=bd6592c37bb2,40087,1732486423713, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=744bbec6934be304e01b009be760323d, server=bd6592c37bb2,40087,1732486423713 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T22:16:09,058 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732486556329.744bbec6934be304e01b009be760323d., hostname=bd6592c37bb2,40087,1732486423713, seqNum=2 because the exception is null or not the one we care about 2024-11-24T22:16:09,058 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40087 {}] ipc.CallRunner(138): callId: 40 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46820 deadline: 1732486629051, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=744bbec6934be304e01b009be760323d, server=bd6592c37bb2,40087,1732486423713 2024-11-24T22:16:09,059 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732486556329.744bbec6934be304e01b009be760323d., hostname=bd6592c37bb2,40087,1732486423713, seqNum=2 , the old value is region=TestAcidGuarantees,,1732486556329.744bbec6934be304e01b009be760323d., hostname=bd6592c37bb2,40087,1732486423713, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=744bbec6934be304e01b009be760323d, server=bd6592c37bb2,40087,1732486423713 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T22:16:09,059 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40087 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=744bbec6934be304e01b009be760323d, server=bd6592c37bb2,40087,1732486423713 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-24T22:16:09,059 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732486556329.744bbec6934be304e01b009be760323d., hostname=bd6592c37bb2,40087,1732486423713, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=744bbec6934be304e01b009be760323d, server=bd6592c37bb2,40087,1732486423713 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T22:16:09,059 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732486556329.744bbec6934be304e01b009be760323d., hostname=bd6592c37bb2,40087,1732486423713, seqNum=2 because the exception is null or not the one we care about 2024-11-24T22:16:09,059 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40087 {}] ipc.CallRunner(138): callId: 44 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46824 deadline: 1732486629051, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=744bbec6934be304e01b009be760323d, server=bd6592c37bb2,40087,1732486423713 2024-11-24T22:16:09,059 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732486556329.744bbec6934be304e01b009be760323d., hostname=bd6592c37bb2,40087,1732486423713, seqNum=2 , the old value is region=TestAcidGuarantees,,1732486556329.744bbec6934be304e01b009be760323d., hostname=bd6592c37bb2,40087,1732486423713, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=744bbec6934be304e01b009be760323d, server=bd6592c37bb2,40087,1732486423713 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T22:16:09,059 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732486556329.744bbec6934be304e01b009be760323d., hostname=bd6592c37bb2,40087,1732486423713, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=744bbec6934be304e01b009be760323d, server=bd6592c37bb2,40087,1732486423713 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T22:16:09,059 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732486556329.744bbec6934be304e01b009be760323d., hostname=bd6592c37bb2,40087,1732486423713, seqNum=2 because the exception is null or not the one we care about 2024-11-24T22:16:09,140 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37701 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=127 2024-11-24T22:16:09,286 INFO [RS_FLUSH_OPERATIONS-regionserver/bd6592c37bb2:0-2 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=38.02 KB at sequenceid=186 (bloomFilter=true), to=hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/744bbec6934be304e01b009be760323d/.tmp/C/d53b3ca00fc140c6b59fdb5c520720c7 2024-11-24T22:16:09,290 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bd6592c37bb2:0-2 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/744bbec6934be304e01b009be760323d/.tmp/A/f98aceb87462436897276c8ce48119a3 as hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/744bbec6934be304e01b009be760323d/A/f98aceb87462436897276c8ce48119a3 2024-11-24T22:16:09,293 INFO [RS_FLUSH_OPERATIONS-regionserver/bd6592c37bb2:0-2 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/744bbec6934be304e01b009be760323d/A/f98aceb87462436897276c8ce48119a3, entries=150, sequenceid=186, filesize=11.9 K 2024-11-24T22:16:09,294 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bd6592c37bb2:0-2 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/744bbec6934be304e01b009be760323d/.tmp/B/858191303eae4384ae680814f2697353 as hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/744bbec6934be304e01b009be760323d/B/858191303eae4384ae680814f2697353 2024-11-24T22:16:09,297 INFO [RS_FLUSH_OPERATIONS-regionserver/bd6592c37bb2:0-2 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/744bbec6934be304e01b009be760323d/B/858191303eae4384ae680814f2697353, entries=150, sequenceid=186, filesize=11.9 K 2024-11-24T22:16:09,297 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bd6592c37bb2:0-2 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/744bbec6934be304e01b009be760323d/.tmp/C/d53b3ca00fc140c6b59fdb5c520720c7 as hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/744bbec6934be304e01b009be760323d/C/d53b3ca00fc140c6b59fdb5c520720c7 2024-11-24T22:16:09,300 INFO [RS_FLUSH_OPERATIONS-regionserver/bd6592c37bb2:0-2 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/744bbec6934be304e01b009be760323d/C/d53b3ca00fc140c6b59fdb5c520720c7, entries=150, sequenceid=186, filesize=11.9 K 2024-11-24T22:16:09,301 INFO [RS_FLUSH_OPERATIONS-regionserver/bd6592c37bb2:0-2 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.HRegion(3140): Finished flush of dataSize ~114.05 KB/116790, heapSize ~339.38 KB/347520, currentSize=60.38 KB/61830 for 744bbec6934be304e01b009be760323d in 1246ms, sequenceid=186, compaction requested=true 2024-11-24T22:16:09,301 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bd6592c37bb2:0-2 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.HRegion(2603): Flush status journal for 744bbec6934be304e01b009be760323d: 2024-11-24T22:16:09,301 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bd6592c37bb2:0-2 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732486556329.744bbec6934be304e01b009be760323d. 2024-11-24T22:16:09,301 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bd6592c37bb2:0-2 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=128 2024-11-24T22:16:09,302 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37701 {}] master.HMaster(4169): Remote procedure done, pid=128 2024-11-24T22:16:09,304 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=128, resume processing ppid=127 2024-11-24T22:16:09,304 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=128, ppid=127, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 2.3170 sec 2024-11-24T22:16:09,305 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=127, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=127, table=TestAcidGuarantees in 2.3220 sec 2024-11-24T22:16:09,316 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40087 {}] regionserver.HRegion(8855): Flush requested on 744bbec6934be304e01b009be760323d 2024-11-24T22:16:09,316 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 744bbec6934be304e01b009be760323d 3/3 column families, dataSize=67.09 KB heapSize=199.97 KB 2024-11-24T22:16:09,317 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 744bbec6934be304e01b009be760323d, store=A 2024-11-24T22:16:09,317 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-24T22:16:09,317 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 744bbec6934be304e01b009be760323d, store=B 2024-11-24T22:16:09,317 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-24T22:16:09,317 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 744bbec6934be304e01b009be760323d, store=C 2024-11-24T22:16:09,317 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-24T22:16:09,321 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/744bbec6934be304e01b009be760323d/.tmp/A/9900ee0a105547bc9d0a8588ea156f77 is 50, key is test_row_0/A:col10/1732486569315/Put/seqid=0 2024-11-24T22:16:09,324 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45929 is added to blk_1073742317_1493 (size=14541) 2024-11-24T22:16:09,401 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40087 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=744bbec6934be304e01b009be760323d, server=bd6592c37bb2,40087,1732486423713 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-24T22:16:09,401 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40087 {}] ipc.CallRunner(138): callId: 37 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46842 deadline: 1732486629400, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=744bbec6934be304e01b009be760323d, server=bd6592c37bb2,40087,1732486423713 2024-11-24T22:16:09,402 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732486556329.744bbec6934be304e01b009be760323d., hostname=bd6592c37bb2,40087,1732486423713, seqNum=2 , the old value is region=TestAcidGuarantees,,1732486556329.744bbec6934be304e01b009be760323d., hostname=bd6592c37bb2,40087,1732486423713, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=744bbec6934be304e01b009be760323d, server=bd6592c37bb2,40087,1732486423713 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T22:16:09,402 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732486556329.744bbec6934be304e01b009be760323d., hostname=bd6592c37bb2,40087,1732486423713, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=744bbec6934be304e01b009be760323d, server=bd6592c37bb2,40087,1732486423713 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T22:16:09,402 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732486556329.744bbec6934be304e01b009be760323d., hostname=bd6592c37bb2,40087,1732486423713, seqNum=2 because the exception is null or not the one we care about 2024-11-24T22:16:09,402 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40087 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=744bbec6934be304e01b009be760323d, server=bd6592c37bb2,40087,1732486423713 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-24T22:16:09,402 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40087 {}] ipc.CallRunner(138): callId: 119 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46792 deadline: 1732486629402, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=744bbec6934be304e01b009be760323d, server=bd6592c37bb2,40087,1732486423713 2024-11-24T22:16:09,403 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732486556329.744bbec6934be304e01b009be760323d., hostname=bd6592c37bb2,40087,1732486423713, seqNum=2 , the old value is region=TestAcidGuarantees,,1732486556329.744bbec6934be304e01b009be760323d., hostname=bd6592c37bb2,40087,1732486423713, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=744bbec6934be304e01b009be760323d, server=bd6592c37bb2,40087,1732486423713 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T22:16:09,403 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732486556329.744bbec6934be304e01b009be760323d., hostname=bd6592c37bb2,40087,1732486423713, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=744bbec6934be304e01b009be760323d, server=bd6592c37bb2,40087,1732486423713 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T22:16:09,403 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732486556329.744bbec6934be304e01b009be760323d., hostname=bd6592c37bb2,40087,1732486423713, seqNum=2 because the exception is null or not the one we care about 2024-11-24T22:16:09,519 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40087 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=744bbec6934be304e01b009be760323d, server=bd6592c37bb2,40087,1732486423713 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-24T22:16:09,519 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40087 {}] ipc.CallRunner(138): callId: 38 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46842 deadline: 1732486629511, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=744bbec6934be304e01b009be760323d, server=bd6592c37bb2,40087,1732486423713 2024-11-24T22:16:09,520 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40087 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=744bbec6934be304e01b009be760323d, server=bd6592c37bb2,40087,1732486423713 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-24T22:16:09,520 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732486556329.744bbec6934be304e01b009be760323d., hostname=bd6592c37bb2,40087,1732486423713, seqNum=2 , the old value is region=TestAcidGuarantees,,1732486556329.744bbec6934be304e01b009be760323d., hostname=bd6592c37bb2,40087,1732486423713, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=744bbec6934be304e01b009be760323d, server=bd6592c37bb2,40087,1732486423713 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T22:16:09,520 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732486556329.744bbec6934be304e01b009be760323d., hostname=bd6592c37bb2,40087,1732486423713, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=744bbec6934be304e01b009be760323d, server=bd6592c37bb2,40087,1732486423713 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T22:16:09,520 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40087 {}] ipc.CallRunner(138): callId: 120 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46792 deadline: 1732486629511, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=744bbec6934be304e01b009be760323d, server=bd6592c37bb2,40087,1732486423713 2024-11-24T22:16:09,520 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732486556329.744bbec6934be304e01b009be760323d., hostname=bd6592c37bb2,40087,1732486423713, seqNum=2 because the exception is null or not the one we care about 2024-11-24T22:16:09,520 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732486556329.744bbec6934be304e01b009be760323d., hostname=bd6592c37bb2,40087,1732486423713, seqNum=2 , the old value is region=TestAcidGuarantees,,1732486556329.744bbec6934be304e01b009be760323d., hostname=bd6592c37bb2,40087,1732486423713, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=744bbec6934be304e01b009be760323d, server=bd6592c37bb2,40087,1732486423713 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T22:16:09,520 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732486556329.744bbec6934be304e01b009be760323d., hostname=bd6592c37bb2,40087,1732486423713, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=744bbec6934be304e01b009be760323d, server=bd6592c37bb2,40087,1732486423713 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T22:16:09,520 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732486556329.744bbec6934be304e01b009be760323d., hostname=bd6592c37bb2,40087,1732486423713, seqNum=2 because the exception is null or not the one we care about 2024-11-24T22:16:09,570 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40087 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=744bbec6934be304e01b009be760323d, server=bd6592c37bb2,40087,1732486423713 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-24T22:16:09,571 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40087 {}] ipc.CallRunner(138): callId: 41 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46820 deadline: 1732486629560, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=744bbec6934be304e01b009be760323d, server=bd6592c37bb2,40087,1732486423713 2024-11-24T22:16:09,571 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732486556329.744bbec6934be304e01b009be760323d., hostname=bd6592c37bb2,40087,1732486423713, seqNum=2 , the old value is region=TestAcidGuarantees,,1732486556329.744bbec6934be304e01b009be760323d., hostname=bd6592c37bb2,40087,1732486423713, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=744bbec6934be304e01b009be760323d, server=bd6592c37bb2,40087,1732486423713 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T22:16:09,571 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732486556329.744bbec6934be304e01b009be760323d., hostname=bd6592c37bb2,40087,1732486423713, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=744bbec6934be304e01b009be760323d, server=bd6592c37bb2,40087,1732486423713 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T22:16:09,571 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732486556329.744bbec6934be304e01b009be760323d., hostname=bd6592c37bb2,40087,1732486423713, seqNum=2 because the exception is null or not the one we care about 2024-11-24T22:16:09,579 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40087 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=744bbec6934be304e01b009be760323d, server=bd6592c37bb2,40087,1732486423713 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-24T22:16:09,579 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40087 {}] ipc.CallRunner(138): callId: 38 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46816 deadline: 1732486629571, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=744bbec6934be304e01b009be760323d, server=bd6592c37bb2,40087,1732486423713 2024-11-24T22:16:09,579 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732486556329.744bbec6934be304e01b009be760323d., hostname=bd6592c37bb2,40087,1732486423713, seqNum=2 , the old value is region=TestAcidGuarantees,,1732486556329.744bbec6934be304e01b009be760323d., hostname=bd6592c37bb2,40087,1732486423713, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=744bbec6934be304e01b009be760323d, server=bd6592c37bb2,40087,1732486423713 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T22:16:09,579 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40087 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=744bbec6934be304e01b009be760323d, server=bd6592c37bb2,40087,1732486423713 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-24T22:16:09,579 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732486556329.744bbec6934be304e01b009be760323d., hostname=bd6592c37bb2,40087,1732486423713, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=744bbec6934be304e01b009be760323d, server=bd6592c37bb2,40087,1732486423713 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T22:16:09,579 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732486556329.744bbec6934be304e01b009be760323d., hostname=bd6592c37bb2,40087,1732486423713, seqNum=2 because the exception is null or not the one we care about 2024-11-24T22:16:09,579 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40087 {}] ipc.CallRunner(138): callId: 45 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46824 deadline: 1732486629571, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=744bbec6934be304e01b009be760323d, server=bd6592c37bb2,40087,1732486423713 2024-11-24T22:16:09,579 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732486556329.744bbec6934be304e01b009be760323d., hostname=bd6592c37bb2,40087,1732486423713, seqNum=2 , the old value is region=TestAcidGuarantees,,1732486556329.744bbec6934be304e01b009be760323d., hostname=bd6592c37bb2,40087,1732486423713, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=744bbec6934be304e01b009be760323d, server=bd6592c37bb2,40087,1732486423713 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T22:16:09,579 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732486556329.744bbec6934be304e01b009be760323d., hostname=bd6592c37bb2,40087,1732486423713, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=744bbec6934be304e01b009be760323d, server=bd6592c37bb2,40087,1732486423713 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T22:16:09,579 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732486556329.744bbec6934be304e01b009be760323d., hostname=bd6592c37bb2,40087,1732486423713, seqNum=2 because the exception is null or not the one we care about 2024-11-24T22:16:09,725 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=199 (bloomFilter=true), to=hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/744bbec6934be304e01b009be760323d/.tmp/A/9900ee0a105547bc9d0a8588ea156f77 2024-11-24T22:16:09,732 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/744bbec6934be304e01b009be760323d/.tmp/B/2baf90c7fd214d82918f0a11a1398b68 is 50, key is test_row_0/B:col10/1732486569315/Put/seqid=0 2024-11-24T22:16:09,735 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45929 is added to blk_1073742318_1494 (size=12151) 2024-11-24T22:16:09,744 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40087 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=744bbec6934be304e01b009be760323d, server=bd6592c37bb2,40087,1732486423713 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-24T22:16:09,745 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40087 {}] ipc.CallRunner(138): callId: 39 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46842 deadline: 1732486629731, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=744bbec6934be304e01b009be760323d, server=bd6592c37bb2,40087,1732486423713 2024-11-24T22:16:09,745 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40087 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=744bbec6934be304e01b009be760323d, server=bd6592c37bb2,40087,1732486423713 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-24T22:16:09,745 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40087 {}] ipc.CallRunner(138): callId: 121 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46792 deadline: 1732486629731, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=744bbec6934be304e01b009be760323d, server=bd6592c37bb2,40087,1732486423713 2024-11-24T22:16:09,745 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732486556329.744bbec6934be304e01b009be760323d., hostname=bd6592c37bb2,40087,1732486423713, seqNum=2 , the old value is region=TestAcidGuarantees,,1732486556329.744bbec6934be304e01b009be760323d., hostname=bd6592c37bb2,40087,1732486423713, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=744bbec6934be304e01b009be760323d, server=bd6592c37bb2,40087,1732486423713 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T22:16:09,745 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732486556329.744bbec6934be304e01b009be760323d., hostname=bd6592c37bb2,40087,1732486423713, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=744bbec6934be304e01b009be760323d, server=bd6592c37bb2,40087,1732486423713 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T22:16:09,745 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732486556329.744bbec6934be304e01b009be760323d., hostname=bd6592c37bb2,40087,1732486423713, seqNum=2 because the exception is null or not the one we care about 2024-11-24T22:16:09,745 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732486556329.744bbec6934be304e01b009be760323d., hostname=bd6592c37bb2,40087,1732486423713, seqNum=2 , the old value is region=TestAcidGuarantees,,1732486556329.744bbec6934be304e01b009be760323d., hostname=bd6592c37bb2,40087,1732486423713, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=744bbec6934be304e01b009be760323d, server=bd6592c37bb2,40087,1732486423713 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T22:16:09,745 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732486556329.744bbec6934be304e01b009be760323d., hostname=bd6592c37bb2,40087,1732486423713, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=744bbec6934be304e01b009be760323d, server=bd6592c37bb2,40087,1732486423713 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T22:16:09,745 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732486556329.744bbec6934be304e01b009be760323d., hostname=bd6592c37bb2,40087,1732486423713, seqNum=2 because the exception is null or not the one we care about 2024-11-24T22:16:10,058 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40087 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=744bbec6934be304e01b009be760323d, server=bd6592c37bb2,40087,1732486423713 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-24T22:16:10,058 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40087 {}] ipc.CallRunner(138): callId: 40 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46842 deadline: 1732486630050, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=744bbec6934be304e01b009be760323d, server=bd6592c37bb2,40087,1732486423713 2024-11-24T22:16:10,059 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40087 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=744bbec6934be304e01b009be760323d, server=bd6592c37bb2,40087,1732486423713 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-24T22:16:10,059 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732486556329.744bbec6934be304e01b009be760323d., hostname=bd6592c37bb2,40087,1732486423713, seqNum=2 , the old value is region=TestAcidGuarantees,,1732486556329.744bbec6934be304e01b009be760323d., hostname=bd6592c37bb2,40087,1732486423713, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=744bbec6934be304e01b009be760323d, server=bd6592c37bb2,40087,1732486423713 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T22:16:10,059 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732486556329.744bbec6934be304e01b009be760323d., hostname=bd6592c37bb2,40087,1732486423713, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=744bbec6934be304e01b009be760323d, server=bd6592c37bb2,40087,1732486423713 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T22:16:10,059 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40087 {}] ipc.CallRunner(138): callId: 122 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46792 deadline: 1732486630051, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=744bbec6934be304e01b009be760323d, server=bd6592c37bb2,40087,1732486423713 2024-11-24T22:16:10,059 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732486556329.744bbec6934be304e01b009be760323d., hostname=bd6592c37bb2,40087,1732486423713, seqNum=2 because the exception is null or not the one we care about 2024-11-24T22:16:10,059 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732486556329.744bbec6934be304e01b009be760323d., hostname=bd6592c37bb2,40087,1732486423713, seqNum=2 , the old value is region=TestAcidGuarantees,,1732486556329.744bbec6934be304e01b009be760323d., hostname=bd6592c37bb2,40087,1732486423713, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=744bbec6934be304e01b009be760323d, server=bd6592c37bb2,40087,1732486423713 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T22:16:10,059 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732486556329.744bbec6934be304e01b009be760323d., hostname=bd6592c37bb2,40087,1732486423713, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=744bbec6934be304e01b009be760323d, server=bd6592c37bb2,40087,1732486423713 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T22:16:10,059 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732486556329.744bbec6934be304e01b009be760323d., hostname=bd6592c37bb2,40087,1732486423713, seqNum=2 because the exception is null or not the one we care about 2024-11-24T22:16:10,136 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=199 (bloomFilter=true), to=hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/744bbec6934be304e01b009be760323d/.tmp/B/2baf90c7fd214d82918f0a11a1398b68 2024-11-24T22:16:10,143 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/744bbec6934be304e01b009be760323d/.tmp/C/73124dd39e54419f99732bf24548a36d is 50, key is test_row_0/C:col10/1732486569315/Put/seqid=0 2024-11-24T22:16:10,146 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45929 is added to blk_1073742319_1495 (size=12151) 2024-11-24T22:16:10,547 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=199 (bloomFilter=true), to=hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/744bbec6934be304e01b009be760323d/.tmp/C/73124dd39e54419f99732bf24548a36d 2024-11-24T22:16:10,551 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/744bbec6934be304e01b009be760323d/.tmp/A/9900ee0a105547bc9d0a8588ea156f77 as hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/744bbec6934be304e01b009be760323d/A/9900ee0a105547bc9d0a8588ea156f77 2024-11-24T22:16:10,554 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/744bbec6934be304e01b009be760323d/A/9900ee0a105547bc9d0a8588ea156f77, entries=200, sequenceid=199, filesize=14.2 K 2024-11-24T22:16:10,555 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/744bbec6934be304e01b009be760323d/.tmp/B/2baf90c7fd214d82918f0a11a1398b68 as hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/744bbec6934be304e01b009be760323d/B/2baf90c7fd214d82918f0a11a1398b68 2024-11-24T22:16:10,557 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/744bbec6934be304e01b009be760323d/B/2baf90c7fd214d82918f0a11a1398b68, entries=150, sequenceid=199, filesize=11.9 K 2024-11-24T22:16:10,558 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/744bbec6934be304e01b009be760323d/.tmp/C/73124dd39e54419f99732bf24548a36d as hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/744bbec6934be304e01b009be760323d/C/73124dd39e54419f99732bf24548a36d 2024-11-24T22:16:10,561 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/744bbec6934be304e01b009be760323d/C/73124dd39e54419f99732bf24548a36d, entries=150, sequenceid=199, filesize=11.9 K 2024-11-24T22:16:10,561 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~67.09 KB/68700, heapSize ~199.92 KB/204720, currentSize=107.34 KB/109920 for 744bbec6934be304e01b009be760323d in 1245ms, sequenceid=199, compaction requested=true 2024-11-24T22:16:10,562 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 744bbec6934be304e01b009be760323d: 2024-11-24T22:16:10,562 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 744bbec6934be304e01b009be760323d:A, priority=-2147483648, current under compaction store size is 1 2024-11-24T22:16:10,562 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-24T22:16:10,562 DEBUG [RS:0;bd6592c37bb2:40087-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-24T22:16:10,562 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 744bbec6934be304e01b009be760323d:B, priority=-2147483648, current under compaction store size is 2 2024-11-24T22:16:10,562 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-24T22:16:10,562 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 744bbec6934be304e01b009be760323d:C, priority=-2147483648, current under compaction store size is 3 2024-11-24T22:16:10,562 DEBUG [RS:0;bd6592c37bb2:40087-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-24T22:16:10,562 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-24T22:16:10,563 DEBUG [RS:0;bd6592c37bb2:40087-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 48946 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-24T22:16:10,563 DEBUG [RS:0;bd6592c37bb2:40087-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 53726 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-24T22:16:10,563 DEBUG [RS:0;bd6592c37bb2:40087-longCompactions-0 {}] regionserver.HStore(1541): 744bbec6934be304e01b009be760323d/B is initiating minor compaction (all files) 2024-11-24T22:16:10,563 DEBUG [RS:0;bd6592c37bb2:40087-shortCompactions-0 {}] regionserver.HStore(1541): 744bbec6934be304e01b009be760323d/A is initiating minor compaction (all files) 2024-11-24T22:16:10,563 INFO [RS:0;bd6592c37bb2:40087-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 744bbec6934be304e01b009be760323d/B in TestAcidGuarantees,,1732486556329.744bbec6934be304e01b009be760323d. 2024-11-24T22:16:10,563 INFO [RS:0;bd6592c37bb2:40087-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 744bbec6934be304e01b009be760323d/A in TestAcidGuarantees,,1732486556329.744bbec6934be304e01b009be760323d. 2024-11-24T22:16:10,563 INFO [RS:0;bd6592c37bb2:40087-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/744bbec6934be304e01b009be760323d/B/39ecf63a40634ca0ac4b613e858c196c, hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/744bbec6934be304e01b009be760323d/B/4bf0ad08615e4d288f198e363bda6c8f, hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/744bbec6934be304e01b009be760323d/B/858191303eae4384ae680814f2697353, hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/744bbec6934be304e01b009be760323d/B/2baf90c7fd214d82918f0a11a1398b68] into tmpdir=hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/744bbec6934be304e01b009be760323d/.tmp, totalSize=47.8 K 2024-11-24T22:16:10,563 INFO [RS:0;bd6592c37bb2:40087-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/744bbec6934be304e01b009be760323d/A/24a77bcdefc14b56820618f3519e5621, hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/744bbec6934be304e01b009be760323d/A/3691c3519de8465991e4b9e6f16216df, hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/744bbec6934be304e01b009be760323d/A/f98aceb87462436897276c8ce48119a3, hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/744bbec6934be304e01b009be760323d/A/9900ee0a105547bc9d0a8588ea156f77] into tmpdir=hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/744bbec6934be304e01b009be760323d/.tmp, totalSize=52.5 K 2024-11-24T22:16:10,563 DEBUG [RS:0;bd6592c37bb2:40087-longCompactions-0 {}] compactions.Compactor(225): Compacting 39ecf63a40634ca0ac4b613e858c196c, keycount=150, bloomtype=ROW, size=12.2 K, encoding=NONE, compression=NONE, seqNum=151, earliestPutTs=1732486564371 2024-11-24T22:16:10,563 DEBUG [RS:0;bd6592c37bb2:40087-shortCompactions-0 {}] compactions.Compactor(225): Compacting 24a77bcdefc14b56820618f3519e5621, keycount=150, bloomtype=ROW, size=12.2 K, encoding=NONE, compression=NONE, seqNum=151, earliestPutTs=1732486564371 2024-11-24T22:16:10,564 DEBUG [RS:0;bd6592c37bb2:40087-longCompactions-0 {}] compactions.Compactor(225): Compacting 4bf0ad08615e4d288f198e363bda6c8f, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=165, earliestPutTs=1732486565074 2024-11-24T22:16:10,564 DEBUG [RS:0;bd6592c37bb2:40087-shortCompactions-0 {}] compactions.Compactor(225): Compacting 3691c3519de8465991e4b9e6f16216df, keycount=200, bloomtype=ROW, size=14.2 K, encoding=NONE, compression=NONE, seqNum=165, earliestPutTs=1732486565055 2024-11-24T22:16:10,564 DEBUG [RS:0;bd6592c37bb2:40087-longCompactions-0 {}] compactions.Compactor(225): Compacting 858191303eae4384ae680814f2697353, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=186, earliestPutTs=1732486567160 2024-11-24T22:16:10,564 DEBUG [RS:0;bd6592c37bb2:40087-shortCompactions-0 {}] compactions.Compactor(225): Compacting f98aceb87462436897276c8ce48119a3, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=186, earliestPutTs=1732486567160 2024-11-24T22:16:10,564 DEBUG [RS:0;bd6592c37bb2:40087-longCompactions-0 {}] compactions.Compactor(225): Compacting 2baf90c7fd214d82918f0a11a1398b68, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=199, earliestPutTs=1732486568393 2024-11-24T22:16:10,564 DEBUG [RS:0;bd6592c37bb2:40087-shortCompactions-0 {}] compactions.Compactor(225): Compacting 9900ee0a105547bc9d0a8588ea156f77, keycount=200, bloomtype=ROW, size=14.2 K, encoding=NONE, compression=NONE, seqNum=199, earliestPutTs=1732486568375 2024-11-24T22:16:10,572 INFO [RS:0;bd6592c37bb2:40087-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 744bbec6934be304e01b009be760323d#A#compaction#417 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 57.69 MB/second 2024-11-24T22:16:10,572 DEBUG [RS:0;bd6592c37bb2:40087-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/744bbec6934be304e01b009be760323d/.tmp/A/2d64710994644bb48471f347abb9a937 is 50, key is test_row_0/A:col10/1732486569315/Put/seqid=0 2024-11-24T22:16:10,573 INFO [RS:0;bd6592c37bb2:40087-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 744bbec6934be304e01b009be760323d#B#compaction#416 average throughput is 2.18 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 57.69 MB/second 2024-11-24T22:16:10,573 DEBUG [RS:0;bd6592c37bb2:40087-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/744bbec6934be304e01b009be760323d/.tmp/B/6cdd2aa199bc4abb9b0c181285f90fb1 is 50, key is test_row_0/B:col10/1732486569315/Put/seqid=0 2024-11-24T22:16:10,576 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45929 is added to blk_1073742320_1496 (size=12629) 2024-11-24T22:16:10,580 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40087 {}] regionserver.HRegion(8855): Flush requested on 744bbec6934be304e01b009be760323d 2024-11-24T22:16:10,580 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 744bbec6934be304e01b009be760323d 3/3 column families, dataSize=114.05 KB heapSize=339.42 KB 2024-11-24T22:16:10,581 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 744bbec6934be304e01b009be760323d, store=A 2024-11-24T22:16:10,581 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-24T22:16:10,581 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 744bbec6934be304e01b009be760323d, store=B 2024-11-24T22:16:10,581 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-24T22:16:10,581 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 744bbec6934be304e01b009be760323d, store=C 2024-11-24T22:16:10,581 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-24T22:16:10,582 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45929 is added to blk_1073742321_1497 (size=12629) 2024-11-24T22:16:10,583 DEBUG [RS:0;bd6592c37bb2:40087-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/744bbec6934be304e01b009be760323d/.tmp/A/2d64710994644bb48471f347abb9a937 as hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/744bbec6934be304e01b009be760323d/A/2d64710994644bb48471f347abb9a937 2024-11-24T22:16:10,587 DEBUG [RS:0;bd6592c37bb2:40087-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/744bbec6934be304e01b009be760323d/.tmp/B/6cdd2aa199bc4abb9b0c181285f90fb1 as hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/744bbec6934be304e01b009be760323d/B/6cdd2aa199bc4abb9b0c181285f90fb1 2024-11-24T22:16:10,588 INFO [RS:0;bd6592c37bb2:40087-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 744bbec6934be304e01b009be760323d/A of 744bbec6934be304e01b009be760323d into 2d64710994644bb48471f347abb9a937(size=12.3 K), total size for store is 12.3 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-24T22:16:10,588 DEBUG [RS:0;bd6592c37bb2:40087-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 744bbec6934be304e01b009be760323d: 2024-11-24T22:16:10,588 INFO [RS:0;bd6592c37bb2:40087-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732486556329.744bbec6934be304e01b009be760323d., storeName=744bbec6934be304e01b009be760323d/A, priority=12, startTime=1732486570562; duration=0sec 2024-11-24T22:16:10,588 DEBUG [RS:0;bd6592c37bb2:40087-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-24T22:16:10,588 DEBUG [RS:0;bd6592c37bb2:40087-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 744bbec6934be304e01b009be760323d:A 2024-11-24T22:16:10,588 DEBUG [RS:0;bd6592c37bb2:40087-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-24T22:16:10,589 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/744bbec6934be304e01b009be760323d/.tmp/A/845bfd56129a4b6fb3107b3ec0c44a38 is 50, key is test_row_0/A:col10/1732486569399/Put/seqid=0 2024-11-24T22:16:10,589 DEBUG [RS:0;bd6592c37bb2:40087-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 48946 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-24T22:16:10,589 DEBUG [RS:0;bd6592c37bb2:40087-shortCompactions-0 {}] regionserver.HStore(1541): 744bbec6934be304e01b009be760323d/C is initiating minor compaction (all files) 2024-11-24T22:16:10,589 INFO [RS:0;bd6592c37bb2:40087-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 744bbec6934be304e01b009be760323d/C in TestAcidGuarantees,,1732486556329.744bbec6934be304e01b009be760323d. 2024-11-24T22:16:10,589 INFO [RS:0;bd6592c37bb2:40087-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/744bbec6934be304e01b009be760323d/C/98ae469563954c268cb17c800408d0ea, hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/744bbec6934be304e01b009be760323d/C/958ad208c5ba4d749d3f1c406bdcd7f7, hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/744bbec6934be304e01b009be760323d/C/d53b3ca00fc140c6b59fdb5c520720c7, hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/744bbec6934be304e01b009be760323d/C/73124dd39e54419f99732bf24548a36d] into tmpdir=hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/744bbec6934be304e01b009be760323d/.tmp, totalSize=47.8 K 2024-11-24T22:16:10,590 DEBUG [RS:0;bd6592c37bb2:40087-shortCompactions-0 {}] compactions.Compactor(225): Compacting 98ae469563954c268cb17c800408d0ea, keycount=150, bloomtype=ROW, size=12.2 K, encoding=NONE, compression=NONE, seqNum=151, earliestPutTs=1732486564371 2024-11-24T22:16:10,590 DEBUG [RS:0;bd6592c37bb2:40087-shortCompactions-0 {}] compactions.Compactor(225): Compacting 958ad208c5ba4d749d3f1c406bdcd7f7, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=165, earliestPutTs=1732486565074 2024-11-24T22:16:10,591 DEBUG [RS:0;bd6592c37bb2:40087-shortCompactions-0 {}] compactions.Compactor(225): Compacting d53b3ca00fc140c6b59fdb5c520720c7, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=186, earliestPutTs=1732486567160 2024-11-24T22:16:10,591 DEBUG [RS:0;bd6592c37bb2:40087-shortCompactions-0 {}] compactions.Compactor(225): Compacting 73124dd39e54419f99732bf24548a36d, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=199, earliestPutTs=1732486568393 2024-11-24T22:16:10,592 INFO [RS:0;bd6592c37bb2:40087-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 744bbec6934be304e01b009be760323d/B of 744bbec6934be304e01b009be760323d into 6cdd2aa199bc4abb9b0c181285f90fb1(size=12.3 K), total size for store is 12.3 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-24T22:16:10,592 DEBUG [RS:0;bd6592c37bb2:40087-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 744bbec6934be304e01b009be760323d: 2024-11-24T22:16:10,592 INFO [RS:0;bd6592c37bb2:40087-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732486556329.744bbec6934be304e01b009be760323d., storeName=744bbec6934be304e01b009be760323d/B, priority=12, startTime=1732486570562; duration=0sec 2024-11-24T22:16:10,592 DEBUG [RS:0;bd6592c37bb2:40087-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-24T22:16:10,592 DEBUG [RS:0;bd6592c37bb2:40087-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 744bbec6934be304e01b009be760323d:B 2024-11-24T22:16:10,593 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45929 is added to blk_1073742322_1498 (size=14541) 2024-11-24T22:16:10,594 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=38.02 KB at sequenceid=219 (bloomFilter=true), to=hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/744bbec6934be304e01b009be760323d/.tmp/A/845bfd56129a4b6fb3107b3ec0c44a38 2024-11-24T22:16:10,602 INFO [RS:0;bd6592c37bb2:40087-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 744bbec6934be304e01b009be760323d#C#compaction#419 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 57.69 MB/second 2024-11-24T22:16:10,602 DEBUG [RS:0;bd6592c37bb2:40087-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/744bbec6934be304e01b009be760323d/.tmp/C/e45e5aa1f5c44dabadda8d63a9a13c5d is 50, key is test_row_0/C:col10/1732486569315/Put/seqid=0 2024-11-24T22:16:10,604 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/744bbec6934be304e01b009be760323d/.tmp/B/015fadc872e249c49869f4546f14e186 is 50, key is test_row_0/B:col10/1732486569399/Put/seqid=0 2024-11-24T22:16:10,607 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45929 is added to blk_1073742323_1499 (size=12629) 2024-11-24T22:16:10,608 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45929 is added to blk_1073742324_1500 (size=12151) 2024-11-24T22:16:10,608 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=38.02 KB at sequenceid=219 (bloomFilter=true), to=hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/744bbec6934be304e01b009be760323d/.tmp/B/015fadc872e249c49869f4546f14e186 2024-11-24T22:16:10,611 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40087 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=744bbec6934be304e01b009be760323d, server=bd6592c37bb2,40087,1732486423713 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-24T22:16:10,611 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40087 {}] ipc.CallRunner(138): callId: 44 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46842 deadline: 1732486630598, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=744bbec6934be304e01b009be760323d, server=bd6592c37bb2,40087,1732486423713 2024-11-24T22:16:10,611 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40087 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=744bbec6934be304e01b009be760323d, server=bd6592c37bb2,40087,1732486423713 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-24T22:16:10,617 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40087 {}] ipc.CallRunner(138): callId: 40 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46816 deadline: 1732486630604, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=744bbec6934be304e01b009be760323d, server=bd6592c37bb2,40087,1732486423713 2024-11-24T22:16:10,617 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732486556329.744bbec6934be304e01b009be760323d., hostname=bd6592c37bb2,40087,1732486423713, seqNum=2 , the old value is region=TestAcidGuarantees,,1732486556329.744bbec6934be304e01b009be760323d., hostname=bd6592c37bb2,40087,1732486423713, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=744bbec6934be304e01b009be760323d, server=bd6592c37bb2,40087,1732486423713 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T22:16:10,617 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732486556329.744bbec6934be304e01b009be760323d., hostname=bd6592c37bb2,40087,1732486423713, seqNum=2 , the old value is region=TestAcidGuarantees,,1732486556329.744bbec6934be304e01b009be760323d., hostname=bd6592c37bb2,40087,1732486423713, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=744bbec6934be304e01b009be760323d, server=bd6592c37bb2,40087,1732486423713 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T22:16:10,617 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732486556329.744bbec6934be304e01b009be760323d., hostname=bd6592c37bb2,40087,1732486423713, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=744bbec6934be304e01b009be760323d, server=bd6592c37bb2,40087,1732486423713 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T22:16:10,617 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732486556329.744bbec6934be304e01b009be760323d., hostname=bd6592c37bb2,40087,1732486423713, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=744bbec6934be304e01b009be760323d, server=bd6592c37bb2,40087,1732486423713 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T22:16:10,617 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732486556329.744bbec6934be304e01b009be760323d., hostname=bd6592c37bb2,40087,1732486423713, seqNum=2 because the exception is null or not the one we care about 2024-11-24T22:16:10,617 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732486556329.744bbec6934be304e01b009be760323d., hostname=bd6592c37bb2,40087,1732486423713, seqNum=2 because the exception is null or not the one we care about 2024-11-24T22:16:10,621 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/744bbec6934be304e01b009be760323d/.tmp/C/ea607d43102d4e1398667662a8d59ffe is 50, key is test_row_0/C:col10/1732486569399/Put/seqid=0 2024-11-24T22:16:10,622 DEBUG [RS:0;bd6592c37bb2:40087-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/744bbec6934be304e01b009be760323d/.tmp/C/e45e5aa1f5c44dabadda8d63a9a13c5d as hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/744bbec6934be304e01b009be760323d/C/e45e5aa1f5c44dabadda8d63a9a13c5d 2024-11-24T22:16:10,625 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45929 is added to blk_1073742325_1501 (size=12151) 2024-11-24T22:16:10,626 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=38.02 KB at sequenceid=219 (bloomFilter=true), to=hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/744bbec6934be304e01b009be760323d/.tmp/C/ea607d43102d4e1398667662a8d59ffe 2024-11-24T22:16:10,626 INFO [RS:0;bd6592c37bb2:40087-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 744bbec6934be304e01b009be760323d/C of 744bbec6934be304e01b009be760323d into e45e5aa1f5c44dabadda8d63a9a13c5d(size=12.3 K), total size for store is 12.3 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-24T22:16:10,626 DEBUG [RS:0;bd6592c37bb2:40087-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 744bbec6934be304e01b009be760323d: 2024-11-24T22:16:10,626 INFO [RS:0;bd6592c37bb2:40087-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732486556329.744bbec6934be304e01b009be760323d., storeName=744bbec6934be304e01b009be760323d/C, priority=12, startTime=1732486570562; duration=0sec 2024-11-24T22:16:10,626 DEBUG [RS:0;bd6592c37bb2:40087-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-24T22:16:10,626 DEBUG [RS:0;bd6592c37bb2:40087-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 744bbec6934be304e01b009be760323d:C 2024-11-24T22:16:10,628 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40087 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=744bbec6934be304e01b009be760323d, server=bd6592c37bb2,40087,1732486423713 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-24T22:16:10,628 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40087 {}] ipc.CallRunner(138): callId: 47 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46824 deadline: 1732486630605, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=744bbec6934be304e01b009be760323d, server=bd6592c37bb2,40087,1732486423713 2024-11-24T22:16:10,629 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40087 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=744bbec6934be304e01b009be760323d, server=bd6592c37bb2,40087,1732486423713 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-24T22:16:10,629 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732486556329.744bbec6934be304e01b009be760323d., hostname=bd6592c37bb2,40087,1732486423713, seqNum=2 , the old value is region=TestAcidGuarantees,,1732486556329.744bbec6934be304e01b009be760323d., hostname=bd6592c37bb2,40087,1732486423713, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=744bbec6934be304e01b009be760323d, server=bd6592c37bb2,40087,1732486423713 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T22:16:10,629 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732486556329.744bbec6934be304e01b009be760323d., hostname=bd6592c37bb2,40087,1732486423713, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=744bbec6934be304e01b009be760323d, server=bd6592c37bb2,40087,1732486423713 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T22:16:10,629 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40087 {}] ipc.CallRunner(138): callId: 44 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46820 deadline: 1732486630607, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=744bbec6934be304e01b009be760323d, server=bd6592c37bb2,40087,1732486423713 2024-11-24T22:16:10,629 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732486556329.744bbec6934be304e01b009be760323d., hostname=bd6592c37bb2,40087,1732486423713, seqNum=2 because the exception is null or not the one we care about 2024-11-24T22:16:10,629 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732486556329.744bbec6934be304e01b009be760323d., hostname=bd6592c37bb2,40087,1732486423713, seqNum=2 , the old value is region=TestAcidGuarantees,,1732486556329.744bbec6934be304e01b009be760323d., hostname=bd6592c37bb2,40087,1732486423713, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=744bbec6934be304e01b009be760323d, server=bd6592c37bb2,40087,1732486423713 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T22:16:10,629 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732486556329.744bbec6934be304e01b009be760323d., hostname=bd6592c37bb2,40087,1732486423713, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=744bbec6934be304e01b009be760323d, server=bd6592c37bb2,40087,1732486423713 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T22:16:10,629 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732486556329.744bbec6934be304e01b009be760323d., hostname=bd6592c37bb2,40087,1732486423713, seqNum=2 because the exception is null or not the one we care about 2024-11-24T22:16:10,630 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/744bbec6934be304e01b009be760323d/.tmp/A/845bfd56129a4b6fb3107b3ec0c44a38 as hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/744bbec6934be304e01b009be760323d/A/845bfd56129a4b6fb3107b3ec0c44a38 2024-11-24T22:16:10,630 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40087 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=744bbec6934be304e01b009be760323d, server=bd6592c37bb2,40087,1732486423713 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-24T22:16:10,631 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40087 {}] ipc.CallRunner(138): callId: 126 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46792 deadline: 1732486630610, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=744bbec6934be304e01b009be760323d, server=bd6592c37bb2,40087,1732486423713 2024-11-24T22:16:10,631 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732486556329.744bbec6934be304e01b009be760323d., hostname=bd6592c37bb2,40087,1732486423713, seqNum=2 , the old value is region=TestAcidGuarantees,,1732486556329.744bbec6934be304e01b009be760323d., hostname=bd6592c37bb2,40087,1732486423713, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=744bbec6934be304e01b009be760323d, server=bd6592c37bb2,40087,1732486423713 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T22:16:10,631 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732486556329.744bbec6934be304e01b009be760323d., hostname=bd6592c37bb2,40087,1732486423713, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=744bbec6934be304e01b009be760323d, server=bd6592c37bb2,40087,1732486423713 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T22:16:10,631 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732486556329.744bbec6934be304e01b009be760323d., hostname=bd6592c37bb2,40087,1732486423713, seqNum=2 because the exception is null or not the one we care about 2024-11-24T22:16:10,635 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/744bbec6934be304e01b009be760323d/A/845bfd56129a4b6fb3107b3ec0c44a38, entries=200, sequenceid=219, filesize=14.2 K 2024-11-24T22:16:10,635 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/744bbec6934be304e01b009be760323d/.tmp/B/015fadc872e249c49869f4546f14e186 as hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/744bbec6934be304e01b009be760323d/B/015fadc872e249c49869f4546f14e186 2024-11-24T22:16:10,638 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/744bbec6934be304e01b009be760323d/B/015fadc872e249c49869f4546f14e186, entries=150, sequenceid=219, filesize=11.9 K 2024-11-24T22:16:10,639 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/744bbec6934be304e01b009be760323d/.tmp/C/ea607d43102d4e1398667662a8d59ffe as hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/744bbec6934be304e01b009be760323d/C/ea607d43102d4e1398667662a8d59ffe 2024-11-24T22:16:10,642 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/744bbec6934be304e01b009be760323d/C/ea607d43102d4e1398667662a8d59ffe, entries=150, sequenceid=219, filesize=11.9 K 2024-11-24T22:16:10,643 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~114.05 KB/116790, heapSize ~339.38 KB/347520, currentSize=60.38 KB/61830 for 744bbec6934be304e01b009be760323d in 63ms, sequenceid=219, compaction requested=false 2024-11-24T22:16:10,643 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 744bbec6934be304e01b009be760323d: 2024-11-24T22:16:10,727 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40087 {}] regionserver.HRegion(8855): Flush requested on 744bbec6934be304e01b009be760323d 2024-11-24T22:16:10,727 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 744bbec6934be304e01b009be760323d 3/3 column families, dataSize=67.09 KB heapSize=199.97 KB 2024-11-24T22:16:10,727 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 744bbec6934be304e01b009be760323d, store=A 2024-11-24T22:16:10,727 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-24T22:16:10,727 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 744bbec6934be304e01b009be760323d, store=B 2024-11-24T22:16:10,727 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-24T22:16:10,727 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 744bbec6934be304e01b009be760323d, store=C 2024-11-24T22:16:10,727 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-24T22:16:10,732 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/744bbec6934be304e01b009be760323d/.tmp/A/f12797d1441c4e36baccfbffe71e8ad9 is 50, key is test_row_0/A:col10/1732486570600/Put/seqid=0 2024-11-24T22:16:10,737 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45929 is added to blk_1073742326_1502 (size=14541) 2024-11-24T22:16:10,773 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40087 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=744bbec6934be304e01b009be760323d, server=bd6592c37bb2,40087,1732486423713 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-24T22:16:10,773 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40087 {}] ipc.CallRunner(138): callId: 51 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46824 deadline: 1732486630759, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=744bbec6934be304e01b009be760323d, server=bd6592c37bb2,40087,1732486423713 2024-11-24T22:16:10,773 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732486556329.744bbec6934be304e01b009be760323d., hostname=bd6592c37bb2,40087,1732486423713, seqNum=2 , the old value is region=TestAcidGuarantees,,1732486556329.744bbec6934be304e01b009be760323d., hostname=bd6592c37bb2,40087,1732486423713, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=744bbec6934be304e01b009be760323d, server=bd6592c37bb2,40087,1732486423713 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T22:16:10,773 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732486556329.744bbec6934be304e01b009be760323d., hostname=bd6592c37bb2,40087,1732486423713, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=744bbec6934be304e01b009be760323d, server=bd6592c37bb2,40087,1732486423713 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T22:16:10,773 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732486556329.744bbec6934be304e01b009be760323d., hostname=bd6592c37bb2,40087,1732486423713, seqNum=2 because the exception is null or not the one we care about 2024-11-24T22:16:10,774 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40087 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=744bbec6934be304e01b009be760323d, server=bd6592c37bb2,40087,1732486423713 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-24T22:16:10,775 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40087 {}] ipc.CallRunner(138): callId: 47 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46820 deadline: 1732486630770, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=744bbec6934be304e01b009be760323d, server=bd6592c37bb2,40087,1732486423713 2024-11-24T22:16:10,775 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732486556329.744bbec6934be304e01b009be760323d., hostname=bd6592c37bb2,40087,1732486423713, seqNum=2 , the old value is region=TestAcidGuarantees,,1732486556329.744bbec6934be304e01b009be760323d., hostname=bd6592c37bb2,40087,1732486423713, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=744bbec6934be304e01b009be760323d, server=bd6592c37bb2,40087,1732486423713 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T22:16:10,775 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732486556329.744bbec6934be304e01b009be760323d., hostname=bd6592c37bb2,40087,1732486423713, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=744bbec6934be304e01b009be760323d, server=bd6592c37bb2,40087,1732486423713 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T22:16:10,775 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732486556329.744bbec6934be304e01b009be760323d., hostname=bd6592c37bb2,40087,1732486423713, seqNum=2 because the exception is null or not the one we care about 2024-11-24T22:16:10,786 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40087 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=744bbec6934be304e01b009be760323d, server=bd6592c37bb2,40087,1732486423713 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-24T22:16:10,786 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40087 {}] ipc.CallRunner(138): callId: 46 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46816 deadline: 1732486630772, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=744bbec6934be304e01b009be760323d, server=bd6592c37bb2,40087,1732486423713 2024-11-24T22:16:10,786 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732486556329.744bbec6934be304e01b009be760323d., hostname=bd6592c37bb2,40087,1732486423713, seqNum=2 , the old value is region=TestAcidGuarantees,,1732486556329.744bbec6934be304e01b009be760323d., hostname=bd6592c37bb2,40087,1732486423713, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=744bbec6934be304e01b009be760323d, server=bd6592c37bb2,40087,1732486423713 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T22:16:10,786 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732486556329.744bbec6934be304e01b009be760323d., hostname=bd6592c37bb2,40087,1732486423713, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=744bbec6934be304e01b009be760323d, server=bd6592c37bb2,40087,1732486423713 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T22:16:10,786 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40087 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=744bbec6934be304e01b009be760323d, server=bd6592c37bb2,40087,1732486423713 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-24T22:16:10,786 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732486556329.744bbec6934be304e01b009be760323d., hostname=bd6592c37bb2,40087,1732486423713, seqNum=2 because the exception is null or not the one we care about 2024-11-24T22:16:10,786 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40087 {}] ipc.CallRunner(138): callId: 129 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46792 deadline: 1732486630772, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=744bbec6934be304e01b009be760323d, server=bd6592c37bb2,40087,1732486423713 2024-11-24T22:16:10,787 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732486556329.744bbec6934be304e01b009be760323d., hostname=bd6592c37bb2,40087,1732486423713, seqNum=2 , the old value is region=TestAcidGuarantees,,1732486556329.744bbec6934be304e01b009be760323d., hostname=bd6592c37bb2,40087,1732486423713, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=744bbec6934be304e01b009be760323d, server=bd6592c37bb2,40087,1732486423713 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T22:16:10,787 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732486556329.744bbec6934be304e01b009be760323d., hostname=bd6592c37bb2,40087,1732486423713, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=744bbec6934be304e01b009be760323d, server=bd6592c37bb2,40087,1732486423713 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T22:16:10,787 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40087 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=744bbec6934be304e01b009be760323d, server=bd6592c37bb2,40087,1732486423713 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-24T22:16:10,787 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732486556329.744bbec6934be304e01b009be760323d., hostname=bd6592c37bb2,40087,1732486423713, seqNum=2 because the exception is null or not the one we care about 2024-11-24T22:16:10,787 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40087 {}] ipc.CallRunner(138): callId: 50 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46842 deadline: 1732486630773, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=744bbec6934be304e01b009be760323d, server=bd6592c37bb2,40087,1732486423713 2024-11-24T22:16:10,787 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732486556329.744bbec6934be304e01b009be760323d., hostname=bd6592c37bb2,40087,1732486423713, seqNum=2 , the old value is region=TestAcidGuarantees,,1732486556329.744bbec6934be304e01b009be760323d., hostname=bd6592c37bb2,40087,1732486423713, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=744bbec6934be304e01b009be760323d, server=bd6592c37bb2,40087,1732486423713 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T22:16:10,787 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732486556329.744bbec6934be304e01b009be760323d., hostname=bd6592c37bb2,40087,1732486423713, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=744bbec6934be304e01b009be760323d, server=bd6592c37bb2,40087,1732486423713 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T22:16:10,787 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732486556329.744bbec6934be304e01b009be760323d., hostname=bd6592c37bb2,40087,1732486423713, seqNum=2 because the exception is null or not the one we care about 2024-11-24T22:16:10,893 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40087 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=744bbec6934be304e01b009be760323d, server=bd6592c37bb2,40087,1732486423713 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-24T22:16:10,893 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40087 {}] ipc.CallRunner(138): callId: 52 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46824 deadline: 1732486630880, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=744bbec6934be304e01b009be760323d, server=bd6592c37bb2,40087,1732486423713 2024-11-24T22:16:10,893 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40087 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=744bbec6934be304e01b009be760323d, server=bd6592c37bb2,40087,1732486423713 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-24T22:16:10,894 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732486556329.744bbec6934be304e01b009be760323d., hostname=bd6592c37bb2,40087,1732486423713, seqNum=2 , the old value is region=TestAcidGuarantees,,1732486556329.744bbec6934be304e01b009be760323d., hostname=bd6592c37bb2,40087,1732486423713, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=744bbec6934be304e01b009be760323d, server=bd6592c37bb2,40087,1732486423713 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T22:16:10,894 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732486556329.744bbec6934be304e01b009be760323d., hostname=bd6592c37bb2,40087,1732486423713, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=744bbec6934be304e01b009be760323d, server=bd6592c37bb2,40087,1732486423713 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T22:16:10,894 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40087 {}] ipc.CallRunner(138): callId: 48 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46820 deadline: 1732486630880, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=744bbec6934be304e01b009be760323d, server=bd6592c37bb2,40087,1732486423713 2024-11-24T22:16:10,894 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732486556329.744bbec6934be304e01b009be760323d., hostname=bd6592c37bb2,40087,1732486423713, seqNum=2 because the exception is null or not the one we care about 2024-11-24T22:16:10,894 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732486556329.744bbec6934be304e01b009be760323d., hostname=bd6592c37bb2,40087,1732486423713, seqNum=2 , the old value is region=TestAcidGuarantees,,1732486556329.744bbec6934be304e01b009be760323d., hostname=bd6592c37bb2,40087,1732486423713, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=744bbec6934be304e01b009be760323d, server=bd6592c37bb2,40087,1732486423713 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T22:16:10,894 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732486556329.744bbec6934be304e01b009be760323d., hostname=bd6592c37bb2,40087,1732486423713, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=744bbec6934be304e01b009be760323d, server=bd6592c37bb2,40087,1732486423713 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T22:16:10,894 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732486556329.744bbec6934be304e01b009be760323d., hostname=bd6592c37bb2,40087,1732486423713, seqNum=2 because the exception is null or not the one we care about 2024-11-24T22:16:10,901 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40087 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=744bbec6934be304e01b009be760323d, server=bd6592c37bb2,40087,1732486423713 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-24T22:16:10,901 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40087 {}] ipc.CallRunner(138): callId: 47 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46816 deadline: 1732486630891, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=744bbec6934be304e01b009be760323d, server=bd6592c37bb2,40087,1732486423713 2024-11-24T22:16:10,901 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732486556329.744bbec6934be304e01b009be760323d., hostname=bd6592c37bb2,40087,1732486423713, seqNum=2 , the old value is region=TestAcidGuarantees,,1732486556329.744bbec6934be304e01b009be760323d., hostname=bd6592c37bb2,40087,1732486423713, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=744bbec6934be304e01b009be760323d, server=bd6592c37bb2,40087,1732486423713 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T22:16:10,901 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40087 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=744bbec6934be304e01b009be760323d, server=bd6592c37bb2,40087,1732486423713 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-24T22:16:10,901 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732486556329.744bbec6934be304e01b009be760323d., hostname=bd6592c37bb2,40087,1732486423713, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=744bbec6934be304e01b009be760323d, server=bd6592c37bb2,40087,1732486423713 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T22:16:10,901 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732486556329.744bbec6934be304e01b009be760323d., hostname=bd6592c37bb2,40087,1732486423713, seqNum=2 because the exception is null or not the one we care about 2024-11-24T22:16:10,901 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40087 {}] ipc.CallRunner(138): callId: 130 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46792 deadline: 1732486630891, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=744bbec6934be304e01b009be760323d, server=bd6592c37bb2,40087,1732486423713 2024-11-24T22:16:10,901 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40087 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=744bbec6934be304e01b009be760323d, server=bd6592c37bb2,40087,1732486423713 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-24T22:16:10,901 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732486556329.744bbec6934be304e01b009be760323d., hostname=bd6592c37bb2,40087,1732486423713, seqNum=2 , the old value is region=TestAcidGuarantees,,1732486556329.744bbec6934be304e01b009be760323d., hostname=bd6592c37bb2,40087,1732486423713, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=744bbec6934be304e01b009be760323d, server=bd6592c37bb2,40087,1732486423713 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T22:16:10,901 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40087 {}] ipc.CallRunner(138): callId: 51 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46842 deadline: 1732486630891, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=744bbec6934be304e01b009be760323d, server=bd6592c37bb2,40087,1732486423713 2024-11-24T22:16:10,901 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732486556329.744bbec6934be304e01b009be760323d., hostname=bd6592c37bb2,40087,1732486423713, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=744bbec6934be304e01b009be760323d, server=bd6592c37bb2,40087,1732486423713 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T22:16:10,901 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732486556329.744bbec6934be304e01b009be760323d., hostname=bd6592c37bb2,40087,1732486423713, seqNum=2 because the exception is null or not the one we care about 2024-11-24T22:16:10,901 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732486556329.744bbec6934be304e01b009be760323d., hostname=bd6592c37bb2,40087,1732486423713, seqNum=2 , the old value is region=TestAcidGuarantees,,1732486556329.744bbec6934be304e01b009be760323d., hostname=bd6592c37bb2,40087,1732486423713, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=744bbec6934be304e01b009be760323d, server=bd6592c37bb2,40087,1732486423713 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T22:16:10,902 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732486556329.744bbec6934be304e01b009be760323d., hostname=bd6592c37bb2,40087,1732486423713, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=744bbec6934be304e01b009be760323d, server=bd6592c37bb2,40087,1732486423713 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T22:16:10,902 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732486556329.744bbec6934be304e01b009be760323d., hostname=bd6592c37bb2,40087,1732486423713, seqNum=2 because the exception is null or not the one we care about 2024-11-24T22:16:11,107 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40087 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=744bbec6934be304e01b009be760323d, server=bd6592c37bb2,40087,1732486423713 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-24T22:16:11,107 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40087 {}] ipc.CallRunner(138): callId: 53 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46824 deadline: 1732486631101, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=744bbec6934be304e01b009be760323d, server=bd6592c37bb2,40087,1732486423713 2024-11-24T22:16:11,107 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40087 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=744bbec6934be304e01b009be760323d, server=bd6592c37bb2,40087,1732486423713 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-24T22:16:11,108 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40087 {}] ipc.CallRunner(138): callId: 49 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46820 deadline: 1732486631101, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=744bbec6934be304e01b009be760323d, server=bd6592c37bb2,40087,1732486423713 2024-11-24T22:16:11,108 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732486556329.744bbec6934be304e01b009be760323d., hostname=bd6592c37bb2,40087,1732486423713, seqNum=2 , the old value is region=TestAcidGuarantees,,1732486556329.744bbec6934be304e01b009be760323d., hostname=bd6592c37bb2,40087,1732486423713, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=744bbec6934be304e01b009be760323d, server=bd6592c37bb2,40087,1732486423713 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T22:16:11,108 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732486556329.744bbec6934be304e01b009be760323d., hostname=bd6592c37bb2,40087,1732486423713, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=744bbec6934be304e01b009be760323d, server=bd6592c37bb2,40087,1732486423713 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T22:16:11,108 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732486556329.744bbec6934be304e01b009be760323d., hostname=bd6592c37bb2,40087,1732486423713, seqNum=2 because the exception is null or not the one we care about 2024-11-24T22:16:11,108 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732486556329.744bbec6934be304e01b009be760323d., hostname=bd6592c37bb2,40087,1732486423713, seqNum=2 , the old value is region=TestAcidGuarantees,,1732486556329.744bbec6934be304e01b009be760323d., hostname=bd6592c37bb2,40087,1732486423713, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=744bbec6934be304e01b009be760323d, server=bd6592c37bb2,40087,1732486423713 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T22:16:11,108 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732486556329.744bbec6934be304e01b009be760323d., hostname=bd6592c37bb2,40087,1732486423713, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=744bbec6934be304e01b009be760323d, server=bd6592c37bb2,40087,1732486423713 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T22:16:11,108 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732486556329.744bbec6934be304e01b009be760323d., hostname=bd6592c37bb2,40087,1732486423713, seqNum=2 because the exception is null or not the one we care about 2024-11-24T22:16:11,122 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40087 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=744bbec6934be304e01b009be760323d, server=bd6592c37bb2,40087,1732486423713 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-24T22:16:11,122 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40087 {}] ipc.CallRunner(138): callId: 48 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46816 deadline: 1732486631111, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=744bbec6934be304e01b009be760323d, server=bd6592c37bb2,40087,1732486423713 2024-11-24T22:16:11,122 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40087 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=744bbec6934be304e01b009be760323d, server=bd6592c37bb2,40087,1732486423713 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-24T22:16:11,123 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732486556329.744bbec6934be304e01b009be760323d., hostname=bd6592c37bb2,40087,1732486423713, seqNum=2 , the old value is region=TestAcidGuarantees,,1732486556329.744bbec6934be304e01b009be760323d., hostname=bd6592c37bb2,40087,1732486423713, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=744bbec6934be304e01b009be760323d, server=bd6592c37bb2,40087,1732486423713 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T22:16:11,123 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732486556329.744bbec6934be304e01b009be760323d., hostname=bd6592c37bb2,40087,1732486423713, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=744bbec6934be304e01b009be760323d, server=bd6592c37bb2,40087,1732486423713 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T22:16:11,123 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732486556329.744bbec6934be304e01b009be760323d., hostname=bd6592c37bb2,40087,1732486423713, seqNum=2 because the exception is null or not the one we care about 2024-11-24T22:16:11,123 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40087 {}] ipc.CallRunner(138): callId: 131 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46792 deadline: 1732486631111, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=744bbec6934be304e01b009be760323d, server=bd6592c37bb2,40087,1732486423713 2024-11-24T22:16:11,123 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40087 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=744bbec6934be304e01b009be760323d, server=bd6592c37bb2,40087,1732486423713 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-24T22:16:11,123 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732486556329.744bbec6934be304e01b009be760323d., hostname=bd6592c37bb2,40087,1732486423713, seqNum=2 , the old value is region=TestAcidGuarantees,,1732486556329.744bbec6934be304e01b009be760323d., hostname=bd6592c37bb2,40087,1732486423713, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=744bbec6934be304e01b009be760323d, server=bd6592c37bb2,40087,1732486423713 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T22:16:11,123 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732486556329.744bbec6934be304e01b009be760323d., hostname=bd6592c37bb2,40087,1732486423713, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=744bbec6934be304e01b009be760323d, server=bd6592c37bb2,40087,1732486423713 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T22:16:11,123 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40087 {}] ipc.CallRunner(138): callId: 52 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46842 deadline: 1732486631111, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=744bbec6934be304e01b009be760323d, server=bd6592c37bb2,40087,1732486423713 2024-11-24T22:16:11,123 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732486556329.744bbec6934be304e01b009be760323d., hostname=bd6592c37bb2,40087,1732486423713, seqNum=2 because the exception is null or not the one we care about 2024-11-24T22:16:11,123 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732486556329.744bbec6934be304e01b009be760323d., hostname=bd6592c37bb2,40087,1732486423713, seqNum=2 , the old value is region=TestAcidGuarantees,,1732486556329.744bbec6934be304e01b009be760323d., hostname=bd6592c37bb2,40087,1732486423713, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=744bbec6934be304e01b009be760323d, server=bd6592c37bb2,40087,1732486423713 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T22:16:11,123 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732486556329.744bbec6934be304e01b009be760323d., hostname=bd6592c37bb2,40087,1732486423713, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=744bbec6934be304e01b009be760323d, server=bd6592c37bb2,40087,1732486423713 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T22:16:11,123 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732486556329.744bbec6934be304e01b009be760323d., hostname=bd6592c37bb2,40087,1732486423713, seqNum=2 because the exception is null or not the one we care about 2024-11-24T22:16:11,138 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=235 (bloomFilter=true), to=hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/744bbec6934be304e01b009be760323d/.tmp/A/f12797d1441c4e36baccfbffe71e8ad9 2024-11-24T22:16:11,145 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/744bbec6934be304e01b009be760323d/.tmp/B/bb1aba0dbb194f9c9e55d4e980089780 is 50, key is test_row_0/B:col10/1732486570600/Put/seqid=0 2024-11-24T22:16:11,148 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45929 is added to blk_1073742327_1503 (size=12151) 2024-11-24T22:16:11,149 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=235 (bloomFilter=true), to=hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/744bbec6934be304e01b009be760323d/.tmp/B/bb1aba0dbb194f9c9e55d4e980089780 2024-11-24T22:16:11,151 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37701 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=127 2024-11-24T22:16:11,151 INFO [RPCClient-NioEventLoopGroup-4-10 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-24T22:16:11,151 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37701 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-24T22:16:11,152 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37701 {}] procedure2.ProcedureExecutor(1139): Stored pid=129, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=129, table=TestAcidGuarantees 2024-11-24T22:16:11,153 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37701 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=129 2024-11-24T22:16:11,153 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=129, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=129, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-24T22:16:11,154 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=129, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=129, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-24T22:16:11,154 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=130, ppid=129, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-24T22:16:11,156 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/744bbec6934be304e01b009be760323d/.tmp/C/21731928879347bfbe3d06c90347654e is 50, key is test_row_0/C:col10/1732486570600/Put/seqid=0 2024-11-24T22:16:11,160 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45929 is added to blk_1073742328_1504 (size=12151) 2024-11-24T22:16:11,160 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=235 (bloomFilter=true), to=hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/744bbec6934be304e01b009be760323d/.tmp/C/21731928879347bfbe3d06c90347654e 2024-11-24T22:16:11,164 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/744bbec6934be304e01b009be760323d/.tmp/A/f12797d1441c4e36baccfbffe71e8ad9 as hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/744bbec6934be304e01b009be760323d/A/f12797d1441c4e36baccfbffe71e8ad9 2024-11-24T22:16:11,166 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/744bbec6934be304e01b009be760323d/A/f12797d1441c4e36baccfbffe71e8ad9, entries=200, sequenceid=235, filesize=14.2 K 2024-11-24T22:16:11,167 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/744bbec6934be304e01b009be760323d/.tmp/B/bb1aba0dbb194f9c9e55d4e980089780 as hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/744bbec6934be304e01b009be760323d/B/bb1aba0dbb194f9c9e55d4e980089780 2024-11-24T22:16:11,170 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/744bbec6934be304e01b009be760323d/B/bb1aba0dbb194f9c9e55d4e980089780, entries=150, sequenceid=235, filesize=11.9 K 2024-11-24T22:16:11,171 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/744bbec6934be304e01b009be760323d/.tmp/C/21731928879347bfbe3d06c90347654e as hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/744bbec6934be304e01b009be760323d/C/21731928879347bfbe3d06c90347654e 2024-11-24T22:16:11,175 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/744bbec6934be304e01b009be760323d/C/21731928879347bfbe3d06c90347654e, entries=150, sequenceid=235, filesize=11.9 K 2024-11-24T22:16:11,176 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~67.09 KB/68700, heapSize ~199.92 KB/204720, currentSize=107.34 KB/109920 for 744bbec6934be304e01b009be760323d in 449ms, sequenceid=235, compaction requested=true 2024-11-24T22:16:11,176 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 744bbec6934be304e01b009be760323d: 2024-11-24T22:16:11,177 DEBUG [RS:0;bd6592c37bb2:40087-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-24T22:16:11,177 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 744bbec6934be304e01b009be760323d:A, priority=-2147483648, current under compaction store size is 1 2024-11-24T22:16:11,177 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-24T22:16:11,177 DEBUG [RS:0;bd6592c37bb2:40087-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-24T22:16:11,177 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 744bbec6934be304e01b009be760323d:B, priority=-2147483648, current under compaction store size is 2 2024-11-24T22:16:11,177 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-24T22:16:11,177 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 744bbec6934be304e01b009be760323d:C, priority=-2147483648, current under compaction store size is 3 2024-11-24T22:16:11,177 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-24T22:16:11,177 DEBUG [RS:0;bd6592c37bb2:40087-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 41711 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-24T22:16:11,178 DEBUG [RS:0;bd6592c37bb2:40087-shortCompactions-0 {}] regionserver.HStore(1541): 744bbec6934be304e01b009be760323d/A is initiating minor compaction (all files) 2024-11-24T22:16:11,178 INFO [RS:0;bd6592c37bb2:40087-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 744bbec6934be304e01b009be760323d/A in TestAcidGuarantees,,1732486556329.744bbec6934be304e01b009be760323d. 2024-11-24T22:16:11,178 INFO [RS:0;bd6592c37bb2:40087-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/744bbec6934be304e01b009be760323d/A/2d64710994644bb48471f347abb9a937, hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/744bbec6934be304e01b009be760323d/A/845bfd56129a4b6fb3107b3ec0c44a38, hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/744bbec6934be304e01b009be760323d/A/f12797d1441c4e36baccfbffe71e8ad9] into tmpdir=hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/744bbec6934be304e01b009be760323d/.tmp, totalSize=40.7 K 2024-11-24T22:16:11,178 DEBUG [RS:0;bd6592c37bb2:40087-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36931 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-24T22:16:11,178 DEBUG [RS:0;bd6592c37bb2:40087-shortCompactions-0 {}] compactions.Compactor(225): Compacting 2d64710994644bb48471f347abb9a937, keycount=150, bloomtype=ROW, size=12.3 K, encoding=NONE, compression=NONE, seqNum=199, earliestPutTs=1732486568393 2024-11-24T22:16:11,178 DEBUG [RS:0;bd6592c37bb2:40087-longCompactions-0 {}] regionserver.HStore(1541): 744bbec6934be304e01b009be760323d/B is initiating minor compaction (all files) 2024-11-24T22:16:11,178 INFO [RS:0;bd6592c37bb2:40087-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 744bbec6934be304e01b009be760323d/B in TestAcidGuarantees,,1732486556329.744bbec6934be304e01b009be760323d. 2024-11-24T22:16:11,178 INFO [RS:0;bd6592c37bb2:40087-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/744bbec6934be304e01b009be760323d/B/6cdd2aa199bc4abb9b0c181285f90fb1, hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/744bbec6934be304e01b009be760323d/B/015fadc872e249c49869f4546f14e186, hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/744bbec6934be304e01b009be760323d/B/bb1aba0dbb194f9c9e55d4e980089780] into tmpdir=hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/744bbec6934be304e01b009be760323d/.tmp, totalSize=36.1 K 2024-11-24T22:16:11,178 DEBUG [RS:0;bd6592c37bb2:40087-shortCompactions-0 {}] compactions.Compactor(225): Compacting 845bfd56129a4b6fb3107b3ec0c44a38, keycount=200, bloomtype=ROW, size=14.2 K, encoding=NONE, compression=NONE, seqNum=219, earliestPutTs=1732486569385 2024-11-24T22:16:11,179 DEBUG [RS:0;bd6592c37bb2:40087-longCompactions-0 {}] compactions.Compactor(225): Compacting 6cdd2aa199bc4abb9b0c181285f90fb1, keycount=150, bloomtype=ROW, size=12.3 K, encoding=NONE, compression=NONE, seqNum=199, earliestPutTs=1732486568393 2024-11-24T22:16:11,179 DEBUG [RS:0;bd6592c37bb2:40087-shortCompactions-0 {}] compactions.Compactor(225): Compacting f12797d1441c4e36baccfbffe71e8ad9, keycount=200, bloomtype=ROW, size=14.2 K, encoding=NONE, compression=NONE, seqNum=235, earliestPutTs=1732486570600 2024-11-24T22:16:11,179 DEBUG [RS:0;bd6592c37bb2:40087-longCompactions-0 {}] compactions.Compactor(225): Compacting 015fadc872e249c49869f4546f14e186, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=219, earliestPutTs=1732486569385 2024-11-24T22:16:11,179 DEBUG [RS:0;bd6592c37bb2:40087-longCompactions-0 {}] compactions.Compactor(225): Compacting bb1aba0dbb194f9c9e55d4e980089780, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=235, earliestPutTs=1732486570600 2024-11-24T22:16:11,188 INFO [RS:0;bd6592c37bb2:40087-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 744bbec6934be304e01b009be760323d#A#compaction#425 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 57.69 MB/second 2024-11-24T22:16:11,189 DEBUG [RS:0;bd6592c37bb2:40087-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/744bbec6934be304e01b009be760323d/.tmp/A/5ad98303c5714401b416c2ba4baaaff5 is 50, key is test_row_0/A:col10/1732486570600/Put/seqid=0 2024-11-24T22:16:11,189 INFO [RS:0;bd6592c37bb2:40087-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 744bbec6934be304e01b009be760323d#B#compaction#426 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 57.69 MB/second 2024-11-24T22:16:11,190 DEBUG [RS:0;bd6592c37bb2:40087-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/744bbec6934be304e01b009be760323d/.tmp/B/dd7740f4d1404c9a8367cc54fe2fbdd3 is 50, key is test_row_0/B:col10/1732486570600/Put/seqid=0 2024-11-24T22:16:11,194 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45929 is added to blk_1073742329_1505 (size=12731) 2024-11-24T22:16:11,196 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45929 is added to blk_1073742330_1506 (size=12731) 2024-11-24T22:16:11,199 DEBUG [RS:0;bd6592c37bb2:40087-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/744bbec6934be304e01b009be760323d/.tmp/A/5ad98303c5714401b416c2ba4baaaff5 as hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/744bbec6934be304e01b009be760323d/A/5ad98303c5714401b416c2ba4baaaff5 2024-11-24T22:16:11,203 INFO [RS:0;bd6592c37bb2:40087-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 744bbec6934be304e01b009be760323d/A of 744bbec6934be304e01b009be760323d into 5ad98303c5714401b416c2ba4baaaff5(size=12.4 K), total size for store is 12.4 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-24T22:16:11,203 DEBUG [RS:0;bd6592c37bb2:40087-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 744bbec6934be304e01b009be760323d: 2024-11-24T22:16:11,204 INFO [RS:0;bd6592c37bb2:40087-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732486556329.744bbec6934be304e01b009be760323d., storeName=744bbec6934be304e01b009be760323d/A, priority=13, startTime=1732486571176; duration=0sec 2024-11-24T22:16:11,204 DEBUG [RS:0;bd6592c37bb2:40087-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-24T22:16:11,204 DEBUG [RS:0;bd6592c37bb2:40087-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 744bbec6934be304e01b009be760323d:A 2024-11-24T22:16:11,204 DEBUG [RS:0;bd6592c37bb2:40087-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-24T22:16:11,205 DEBUG [RS:0;bd6592c37bb2:40087-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36931 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-24T22:16:11,205 DEBUG [RS:0;bd6592c37bb2:40087-shortCompactions-0 {}] regionserver.HStore(1541): 744bbec6934be304e01b009be760323d/C is initiating minor compaction (all files) 2024-11-24T22:16:11,205 INFO [RS:0;bd6592c37bb2:40087-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 744bbec6934be304e01b009be760323d/C in TestAcidGuarantees,,1732486556329.744bbec6934be304e01b009be760323d. 2024-11-24T22:16:11,206 INFO [RS:0;bd6592c37bb2:40087-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/744bbec6934be304e01b009be760323d/C/e45e5aa1f5c44dabadda8d63a9a13c5d, hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/744bbec6934be304e01b009be760323d/C/ea607d43102d4e1398667662a8d59ffe, hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/744bbec6934be304e01b009be760323d/C/21731928879347bfbe3d06c90347654e] into tmpdir=hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/744bbec6934be304e01b009be760323d/.tmp, totalSize=36.1 K 2024-11-24T22:16:11,206 DEBUG [RS:0;bd6592c37bb2:40087-shortCompactions-0 {}] compactions.Compactor(225): Compacting e45e5aa1f5c44dabadda8d63a9a13c5d, keycount=150, bloomtype=ROW, size=12.3 K, encoding=NONE, compression=NONE, seqNum=199, earliestPutTs=1732486568393 2024-11-24T22:16:11,206 DEBUG [RS:0;bd6592c37bb2:40087-shortCompactions-0 {}] compactions.Compactor(225): Compacting ea607d43102d4e1398667662a8d59ffe, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=219, earliestPutTs=1732486569385 2024-11-24T22:16:11,206 DEBUG [RS:0;bd6592c37bb2:40087-shortCompactions-0 {}] compactions.Compactor(225): Compacting 21731928879347bfbe3d06c90347654e, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=235, earliestPutTs=1732486570600 2024-11-24T22:16:11,213 INFO [RS:0;bd6592c37bb2:40087-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 744bbec6934be304e01b009be760323d#C#compaction#427 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 57.69 MB/second 2024-11-24T22:16:11,214 DEBUG [RS:0;bd6592c37bb2:40087-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/744bbec6934be304e01b009be760323d/.tmp/C/fd0b85f893e14463833221db0d0fcbf9 is 50, key is test_row_0/C:col10/1732486570600/Put/seqid=0 2024-11-24T22:16:11,217 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45929 is added to blk_1073742331_1507 (size=12731) 2024-11-24T22:16:11,261 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37701 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=129 2024-11-24T22:16:11,306 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=40087 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=130 2024-11-24T22:16:11,306 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bd6592c37bb2:0-0 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732486556329.744bbec6934be304e01b009be760323d. 2024-11-24T22:16:11,307 INFO [RS_FLUSH_OPERATIONS-regionserver/bd6592c37bb2:0-0 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.HRegion(2902): Flushing 744bbec6934be304e01b009be760323d 3/3 column families, dataSize=107.34 KB heapSize=319.50 KB 2024-11-24T22:16:11,307 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bd6592c37bb2:0-0 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 744bbec6934be304e01b009be760323d, store=A 2024-11-24T22:16:11,307 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bd6592c37bb2:0-0 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-24T22:16:11,307 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bd6592c37bb2:0-0 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 744bbec6934be304e01b009be760323d, store=B 2024-11-24T22:16:11,307 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bd6592c37bb2:0-0 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-24T22:16:11,307 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bd6592c37bb2:0-0 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 744bbec6934be304e01b009be760323d, store=C 2024-11-24T22:16:11,307 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bd6592c37bb2:0-0 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-24T22:16:11,312 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bd6592c37bb2:0-0 {event_type=RS_FLUSH_REGIONS, pid=130}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/744bbec6934be304e01b009be760323d/.tmp/A/6b1fe8f1a1474845bad917d19afeb88c is 50, key is test_row_0/A:col10/1732486570771/Put/seqid=0 2024-11-24T22:16:11,315 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45929 is added to blk_1073742332_1508 (size=12151) 2024-11-24T22:16:11,415 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40087 {}] regionserver.HRegion(8855): Flush requested on 744bbec6934be304e01b009be760323d 2024-11-24T22:16:11,416 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732486556329.744bbec6934be304e01b009be760323d. as already flushing 2024-11-24T22:16:11,446 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40087 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=744bbec6934be304e01b009be760323d, server=bd6592c37bb2,40087,1732486423713 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-24T22:16:11,446 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40087 {}] ipc.CallRunner(138): callId: 58 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46824 deadline: 1732486631437, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=744bbec6934be304e01b009be760323d, server=bd6592c37bb2,40087,1732486423713 2024-11-24T22:16:11,446 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732486556329.744bbec6934be304e01b009be760323d., hostname=bd6592c37bb2,40087,1732486423713, seqNum=2 , the old value is region=TestAcidGuarantees,,1732486556329.744bbec6934be304e01b009be760323d., hostname=bd6592c37bb2,40087,1732486423713, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=744bbec6934be304e01b009be760323d, server=bd6592c37bb2,40087,1732486423713 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T22:16:11,447 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732486556329.744bbec6934be304e01b009be760323d., hostname=bd6592c37bb2,40087,1732486423713, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=744bbec6934be304e01b009be760323d, server=bd6592c37bb2,40087,1732486423713 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T22:16:11,447 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732486556329.744bbec6934be304e01b009be760323d., hostname=bd6592c37bb2,40087,1732486423713, seqNum=2 because the exception is null or not the one we care about 2024-11-24T22:16:11,448 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40087 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=744bbec6934be304e01b009be760323d, server=bd6592c37bb2,40087,1732486423713 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-24T22:16:11,448 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40087 {}] ipc.CallRunner(138): callId: 50 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46816 deadline: 1732486631443, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=744bbec6934be304e01b009be760323d, server=bd6592c37bb2,40087,1732486423713 2024-11-24T22:16:11,448 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732486556329.744bbec6934be304e01b009be760323d., hostname=bd6592c37bb2,40087,1732486423713, seqNum=2 , the old value is region=TestAcidGuarantees,,1732486556329.744bbec6934be304e01b009be760323d., hostname=bd6592c37bb2,40087,1732486423713, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=744bbec6934be304e01b009be760323d, server=bd6592c37bb2,40087,1732486423713 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T22:16:11,449 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732486556329.744bbec6934be304e01b009be760323d., hostname=bd6592c37bb2,40087,1732486423713, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=744bbec6934be304e01b009be760323d, server=bd6592c37bb2,40087,1732486423713 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T22:16:11,449 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732486556329.744bbec6934be304e01b009be760323d., hostname=bd6592c37bb2,40087,1732486423713, seqNum=2 because the exception is null or not the one we care about 2024-11-24T22:16:11,458 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40087 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=744bbec6934be304e01b009be760323d, server=bd6592c37bb2,40087,1732486423713 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-24T22:16:11,458 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40087 {}] ipc.CallRunner(138): callId: 54 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46842 deadline: 1732486631444, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=744bbec6934be304e01b009be760323d, server=bd6592c37bb2,40087,1732486423713 2024-11-24T22:16:11,459 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40087 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=744bbec6934be304e01b009be760323d, server=bd6592c37bb2,40087,1732486423713 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-24T22:16:11,459 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732486556329.744bbec6934be304e01b009be760323d., hostname=bd6592c37bb2,40087,1732486423713, seqNum=2 , the old value is region=TestAcidGuarantees,,1732486556329.744bbec6934be304e01b009be760323d., hostname=bd6592c37bb2,40087,1732486423713, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=744bbec6934be304e01b009be760323d, server=bd6592c37bb2,40087,1732486423713 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T22:16:11,459 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40087 {}] ipc.CallRunner(138): callId: 133 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46792 deadline: 1732486631445, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=744bbec6934be304e01b009be760323d, server=bd6592c37bb2,40087,1732486423713 2024-11-24T22:16:11,459 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732486556329.744bbec6934be304e01b009be760323d., hostname=bd6592c37bb2,40087,1732486423713, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=744bbec6934be304e01b009be760323d, server=bd6592c37bb2,40087,1732486423713 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T22:16:11,459 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732486556329.744bbec6934be304e01b009be760323d., hostname=bd6592c37bb2,40087,1732486423713, seqNum=2 because the exception is null or not the one we care about 2024-11-24T22:16:11,459 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40087 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=744bbec6934be304e01b009be760323d, server=bd6592c37bb2,40087,1732486423713 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-24T22:16:11,459 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732486556329.744bbec6934be304e01b009be760323d., hostname=bd6592c37bb2,40087,1732486423713, seqNum=2 , the old value is region=TestAcidGuarantees,,1732486556329.744bbec6934be304e01b009be760323d., hostname=bd6592c37bb2,40087,1732486423713, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=744bbec6934be304e01b009be760323d, server=bd6592c37bb2,40087,1732486423713 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T22:16:11,459 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732486556329.744bbec6934be304e01b009be760323d., hostname=bd6592c37bb2,40087,1732486423713, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=744bbec6934be304e01b009be760323d, server=bd6592c37bb2,40087,1732486423713 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T22:16:11,459 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40087 {}] ipc.CallRunner(138): callId: 54 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46820 deadline: 1732486631446, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=744bbec6934be304e01b009be760323d, server=bd6592c37bb2,40087,1732486423713 2024-11-24T22:16:11,459 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732486556329.744bbec6934be304e01b009be760323d., hostname=bd6592c37bb2,40087,1732486423713, seqNum=2 because the exception is null or not the one we care about 2024-11-24T22:16:11,459 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732486556329.744bbec6934be304e01b009be760323d., hostname=bd6592c37bb2,40087,1732486423713, seqNum=2 , the old value is region=TestAcidGuarantees,,1732486556329.744bbec6934be304e01b009be760323d., hostname=bd6592c37bb2,40087,1732486423713, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=744bbec6934be304e01b009be760323d, server=bd6592c37bb2,40087,1732486423713 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T22:16:11,459 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732486556329.744bbec6934be304e01b009be760323d., hostname=bd6592c37bb2,40087,1732486423713, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=744bbec6934be304e01b009be760323d, server=bd6592c37bb2,40087,1732486423713 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T22:16:11,459 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732486556329.744bbec6934be304e01b009be760323d., hostname=bd6592c37bb2,40087,1732486423713, seqNum=2 because the exception is null or not the one we care about 2024-11-24T22:16:11,471 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37701 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=129 2024-11-24T22:16:11,560 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40087 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=744bbec6934be304e01b009be760323d, server=bd6592c37bb2,40087,1732486423713 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-24T22:16:11,560 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40087 {}] ipc.CallRunner(138): callId: 59 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46824 deadline: 1732486631550, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=744bbec6934be304e01b009be760323d, server=bd6592c37bb2,40087,1732486423713 2024-11-24T22:16:11,560 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732486556329.744bbec6934be304e01b009be760323d., hostname=bd6592c37bb2,40087,1732486423713, seqNum=2 , the old value is region=TestAcidGuarantees,,1732486556329.744bbec6934be304e01b009be760323d., hostname=bd6592c37bb2,40087,1732486423713, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=744bbec6934be304e01b009be760323d, server=bd6592c37bb2,40087,1732486423713 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T22:16:11,560 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40087 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=744bbec6934be304e01b009be760323d, server=bd6592c37bb2,40087,1732486423713 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-24T22:16:11,560 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732486556329.744bbec6934be304e01b009be760323d., hostname=bd6592c37bb2,40087,1732486423713, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=744bbec6934be304e01b009be760323d, server=bd6592c37bb2,40087,1732486423713 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T22:16:11,560 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732486556329.744bbec6934be304e01b009be760323d., hostname=bd6592c37bb2,40087,1732486423713, seqNum=2 because the exception is null or not the one we care about 2024-11-24T22:16:11,560 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40087 {}] ipc.CallRunner(138): callId: 51 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46816 deadline: 1732486631550, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=744bbec6934be304e01b009be760323d, server=bd6592c37bb2,40087,1732486423713 2024-11-24T22:16:11,561 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732486556329.744bbec6934be304e01b009be760323d., hostname=bd6592c37bb2,40087,1732486423713, seqNum=2 , the old value is region=TestAcidGuarantees,,1732486556329.744bbec6934be304e01b009be760323d., hostname=bd6592c37bb2,40087,1732486423713, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=744bbec6934be304e01b009be760323d, server=bd6592c37bb2,40087,1732486423713 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T22:16:11,561 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732486556329.744bbec6934be304e01b009be760323d., hostname=bd6592c37bb2,40087,1732486423713, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=744bbec6934be304e01b009be760323d, server=bd6592c37bb2,40087,1732486423713 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T22:16:11,561 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732486556329.744bbec6934be304e01b009be760323d., hostname=bd6592c37bb2,40087,1732486423713, seqNum=2 because the exception is null or not the one we care about 2024-11-24T22:16:11,571 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40087 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=744bbec6934be304e01b009be760323d, server=bd6592c37bb2,40087,1732486423713 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-24T22:16:11,571 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40087 {}] ipc.CallRunner(138): callId: 55 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46842 deadline: 1732486631561, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=744bbec6934be304e01b009be760323d, server=bd6592c37bb2,40087,1732486423713 2024-11-24T22:16:11,572 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732486556329.744bbec6934be304e01b009be760323d., hostname=bd6592c37bb2,40087,1732486423713, seqNum=2 , the old value is region=TestAcidGuarantees,,1732486556329.744bbec6934be304e01b009be760323d., hostname=bd6592c37bb2,40087,1732486423713, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=744bbec6934be304e01b009be760323d, server=bd6592c37bb2,40087,1732486423713 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T22:16:11,572 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732486556329.744bbec6934be304e01b009be760323d., hostname=bd6592c37bb2,40087,1732486423713, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=744bbec6934be304e01b009be760323d, server=bd6592c37bb2,40087,1732486423713 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T22:16:11,572 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732486556329.744bbec6934be304e01b009be760323d., hostname=bd6592c37bb2,40087,1732486423713, seqNum=2 because the exception is null or not the one we care about 2024-11-24T22:16:11,580 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40087 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=744bbec6934be304e01b009be760323d, server=bd6592c37bb2,40087,1732486423713 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-24T22:16:11,580 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40087 {}] ipc.CallRunner(138): callId: 134 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46792 deadline: 1732486631571, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=744bbec6934be304e01b009be760323d, server=bd6592c37bb2,40087,1732486423713 2024-11-24T22:16:11,580 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732486556329.744bbec6934be304e01b009be760323d., hostname=bd6592c37bb2,40087,1732486423713, seqNum=2 , the old value is region=TestAcidGuarantees,,1732486556329.744bbec6934be304e01b009be760323d., hostname=bd6592c37bb2,40087,1732486423713, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=744bbec6934be304e01b009be760323d, server=bd6592c37bb2,40087,1732486423713 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T22:16:11,580 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40087 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=744bbec6934be304e01b009be760323d, server=bd6592c37bb2,40087,1732486423713 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-24T22:16:11,580 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732486556329.744bbec6934be304e01b009be760323d., hostname=bd6592c37bb2,40087,1732486423713, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=744bbec6934be304e01b009be760323d, server=bd6592c37bb2,40087,1732486423713 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T22:16:11,580 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732486556329.744bbec6934be304e01b009be760323d., hostname=bd6592c37bb2,40087,1732486423713, seqNum=2 because the exception is null or not the one we care about 2024-11-24T22:16:11,580 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40087 {}] ipc.CallRunner(138): callId: 55 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46820 deadline: 1732486631571, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=744bbec6934be304e01b009be760323d, server=bd6592c37bb2,40087,1732486423713 2024-11-24T22:16:11,580 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732486556329.744bbec6934be304e01b009be760323d., hostname=bd6592c37bb2,40087,1732486423713, seqNum=2 , the old value is region=TestAcidGuarantees,,1732486556329.744bbec6934be304e01b009be760323d., hostname=bd6592c37bb2,40087,1732486423713, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=744bbec6934be304e01b009be760323d, server=bd6592c37bb2,40087,1732486423713 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T22:16:11,580 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732486556329.744bbec6934be304e01b009be760323d., hostname=bd6592c37bb2,40087,1732486423713, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=744bbec6934be304e01b009be760323d, server=bd6592c37bb2,40087,1732486423713 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T22:16:11,580 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732486556329.744bbec6934be304e01b009be760323d., hostname=bd6592c37bb2,40087,1732486423713, seqNum=2 because the exception is null or not the one we care about 2024-11-24T22:16:11,600 DEBUG [RS:0;bd6592c37bb2:40087-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/744bbec6934be304e01b009be760323d/.tmp/B/dd7740f4d1404c9a8367cc54fe2fbdd3 as hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/744bbec6934be304e01b009be760323d/B/dd7740f4d1404c9a8367cc54fe2fbdd3 2024-11-24T22:16:11,604 INFO [RS:0;bd6592c37bb2:40087-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 744bbec6934be304e01b009be760323d/B of 744bbec6934be304e01b009be760323d into dd7740f4d1404c9a8367cc54fe2fbdd3(size=12.4 K), total size for store is 12.4 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-24T22:16:11,604 DEBUG [RS:0;bd6592c37bb2:40087-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 744bbec6934be304e01b009be760323d: 2024-11-24T22:16:11,604 INFO [RS:0;bd6592c37bb2:40087-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732486556329.744bbec6934be304e01b009be760323d., storeName=744bbec6934be304e01b009be760323d/B, priority=13, startTime=1732486571177; duration=0sec 2024-11-24T22:16:11,604 DEBUG [RS:0;bd6592c37bb2:40087-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-24T22:16:11,604 DEBUG [RS:0;bd6592c37bb2:40087-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 744bbec6934be304e01b009be760323d:B 2024-11-24T22:16:11,621 DEBUG [RS:0;bd6592c37bb2:40087-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/744bbec6934be304e01b009be760323d/.tmp/C/fd0b85f893e14463833221db0d0fcbf9 as hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/744bbec6934be304e01b009be760323d/C/fd0b85f893e14463833221db0d0fcbf9 2024-11-24T22:16:11,624 INFO [RS:0;bd6592c37bb2:40087-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 744bbec6934be304e01b009be760323d/C of 744bbec6934be304e01b009be760323d into fd0b85f893e14463833221db0d0fcbf9(size=12.4 K), total size for store is 12.4 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-24T22:16:11,624 DEBUG [RS:0;bd6592c37bb2:40087-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 744bbec6934be304e01b009be760323d: 2024-11-24T22:16:11,625 INFO [RS:0;bd6592c37bb2:40087-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732486556329.744bbec6934be304e01b009be760323d., storeName=744bbec6934be304e01b009be760323d/C, priority=13, startTime=1732486571177; duration=0sec 2024-11-24T22:16:11,625 DEBUG [RS:0;bd6592c37bb2:40087-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-24T22:16:11,625 DEBUG [RS:0;bd6592c37bb2:40087-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 744bbec6934be304e01b009be760323d:C 2024-11-24T22:16:11,715 INFO [RS_FLUSH_OPERATIONS-regionserver/bd6592c37bb2:0-0 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=35.78 KB at sequenceid=255 (bloomFilter=true), to=hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/744bbec6934be304e01b009be760323d/.tmp/A/6b1fe8f1a1474845bad917d19afeb88c 2024-11-24T22:16:11,722 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bd6592c37bb2:0-0 {event_type=RS_FLUSH_REGIONS, pid=130}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/744bbec6934be304e01b009be760323d/.tmp/B/58027f2e64c3466da9d9ec87e36f74e6 is 50, key is test_row_0/B:col10/1732486570771/Put/seqid=0 2024-11-24T22:16:11,728 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45929 is added to blk_1073742333_1509 (size=12151) 2024-11-24T22:16:11,733 INFO [RS_FLUSH_OPERATIONS-regionserver/bd6592c37bb2:0-0 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=35.78 KB at sequenceid=255 (bloomFilter=true), to=hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/744bbec6934be304e01b009be760323d/.tmp/B/58027f2e64c3466da9d9ec87e36f74e6 2024-11-24T22:16:11,740 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bd6592c37bb2:0-0 {event_type=RS_FLUSH_REGIONS, pid=130}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/744bbec6934be304e01b009be760323d/.tmp/C/90db8ce5d3d54446be80c89ee834828f is 50, key is test_row_0/C:col10/1732486570771/Put/seqid=0 2024-11-24T22:16:11,743 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45929 is added to blk_1073742334_1510 (size=12151) 2024-11-24T22:16:11,776 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40087 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=744bbec6934be304e01b009be760323d, server=bd6592c37bb2,40087,1732486423713 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-24T22:16:11,777 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40087 {}] ipc.CallRunner(138): callId: 60 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46824 deadline: 1732486631770, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=744bbec6934be304e01b009be760323d, server=bd6592c37bb2,40087,1732486423713 2024-11-24T22:16:11,777 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40087 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=744bbec6934be304e01b009be760323d, server=bd6592c37bb2,40087,1732486423713 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-24T22:16:11,777 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732486556329.744bbec6934be304e01b009be760323d., hostname=bd6592c37bb2,40087,1732486423713, seqNum=2 , the old value is region=TestAcidGuarantees,,1732486556329.744bbec6934be304e01b009be760323d., hostname=bd6592c37bb2,40087,1732486423713, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=744bbec6934be304e01b009be760323d, server=bd6592c37bb2,40087,1732486423713 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T22:16:11,777 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40087 {}] ipc.CallRunner(138): callId: 52 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46816 deadline: 1732486631770, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=744bbec6934be304e01b009be760323d, server=bd6592c37bb2,40087,1732486423713 2024-11-24T22:16:11,777 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732486556329.744bbec6934be304e01b009be760323d., hostname=bd6592c37bb2,40087,1732486423713, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=744bbec6934be304e01b009be760323d, server=bd6592c37bb2,40087,1732486423713 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T22:16:11,777 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732486556329.744bbec6934be304e01b009be760323d., hostname=bd6592c37bb2,40087,1732486423713, seqNum=2 because the exception is null or not the one we care about 2024-11-24T22:16:11,777 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732486556329.744bbec6934be304e01b009be760323d., hostname=bd6592c37bb2,40087,1732486423713, seqNum=2 , the old value is region=TestAcidGuarantees,,1732486556329.744bbec6934be304e01b009be760323d., hostname=bd6592c37bb2,40087,1732486423713, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=744bbec6934be304e01b009be760323d, server=bd6592c37bb2,40087,1732486423713 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T22:16:11,777 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732486556329.744bbec6934be304e01b009be760323d., hostname=bd6592c37bb2,40087,1732486423713, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=744bbec6934be304e01b009be760323d, server=bd6592c37bb2,40087,1732486423713 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T22:16:11,777 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732486556329.744bbec6934be304e01b009be760323d., hostname=bd6592c37bb2,40087,1732486423713, seqNum=2 because the exception is null or not the one we care about 2024-11-24T22:16:11,781 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37701 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=129 2024-11-24T22:16:11,788 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40087 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=744bbec6934be304e01b009be760323d, server=bd6592c37bb2,40087,1732486423713 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-24T22:16:11,788 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40087 {}] ipc.CallRunner(138): callId: 56 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46842 deadline: 1732486631781, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=744bbec6934be304e01b009be760323d, server=bd6592c37bb2,40087,1732486423713 2024-11-24T22:16:11,788 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732486556329.744bbec6934be304e01b009be760323d., hostname=bd6592c37bb2,40087,1732486423713, seqNum=2 , the old value is region=TestAcidGuarantees,,1732486556329.744bbec6934be304e01b009be760323d., hostname=bd6592c37bb2,40087,1732486423713, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=744bbec6934be304e01b009be760323d, server=bd6592c37bb2,40087,1732486423713 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T22:16:11,788 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732486556329.744bbec6934be304e01b009be760323d., hostname=bd6592c37bb2,40087,1732486423713, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=744bbec6934be304e01b009be760323d, server=bd6592c37bb2,40087,1732486423713 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T22:16:11,788 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732486556329.744bbec6934be304e01b009be760323d., hostname=bd6592c37bb2,40087,1732486423713, seqNum=2 because the exception is null or not the one we care about 2024-11-24T22:16:11,801 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40087 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=744bbec6934be304e01b009be760323d, server=bd6592c37bb2,40087,1732486423713 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-24T22:16:11,801 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40087 {}] ipc.CallRunner(138): callId: 135 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46792 deadline: 1732486631791, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=744bbec6934be304e01b009be760323d, server=bd6592c37bb2,40087,1732486423713 2024-11-24T22:16:11,802 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732486556329.744bbec6934be304e01b009be760323d., hostname=bd6592c37bb2,40087,1732486423713, seqNum=2 , the old value is region=TestAcidGuarantees,,1732486556329.744bbec6934be304e01b009be760323d., hostname=bd6592c37bb2,40087,1732486423713, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=744bbec6934be304e01b009be760323d, server=bd6592c37bb2,40087,1732486423713 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T22:16:11,802 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40087 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=744bbec6934be304e01b009be760323d, server=bd6592c37bb2,40087,1732486423713 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-24T22:16:11,802 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732486556329.744bbec6934be304e01b009be760323d., hostname=bd6592c37bb2,40087,1732486423713, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=744bbec6934be304e01b009be760323d, server=bd6592c37bb2,40087,1732486423713 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T22:16:11,802 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732486556329.744bbec6934be304e01b009be760323d., hostname=bd6592c37bb2,40087,1732486423713, seqNum=2 because the exception is null or not the one we care about 2024-11-24T22:16:11,802 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40087 {}] ipc.CallRunner(138): callId: 56 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46820 deadline: 1732486631791, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=744bbec6934be304e01b009be760323d, server=bd6592c37bb2,40087,1732486423713 2024-11-24T22:16:11,802 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732486556329.744bbec6934be304e01b009be760323d., hostname=bd6592c37bb2,40087,1732486423713, seqNum=2 , the old value is region=TestAcidGuarantees,,1732486556329.744bbec6934be304e01b009be760323d., hostname=bd6592c37bb2,40087,1732486423713, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=744bbec6934be304e01b009be760323d, server=bd6592c37bb2,40087,1732486423713 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T22:16:11,802 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732486556329.744bbec6934be304e01b009be760323d., hostname=bd6592c37bb2,40087,1732486423713, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=744bbec6934be304e01b009be760323d, server=bd6592c37bb2,40087,1732486423713 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T22:16:11,802 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732486556329.744bbec6934be304e01b009be760323d., hostname=bd6592c37bb2,40087,1732486423713, seqNum=2 because the exception is null or not the one we care about 2024-11-24T22:16:12,015 DEBUG [FsDatasetAsyncDiskServiceFixer {}] hbase.HBaseTestingUtil$FsDatasetAsyncDiskServiceFixer(597): NoSuchFieldException: threadGroup; It might because your Hadoop version > 3.2.3 or 3.3.4, See HBASE-27595 for details. 2024-11-24T22:16:12,087 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40087 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=744bbec6934be304e01b009be760323d, server=bd6592c37bb2,40087,1732486423713 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-24T22:16:12,087 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40087 {}] ipc.CallRunner(138): callId: 61 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46824 deadline: 1732486632081, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=744bbec6934be304e01b009be760323d, server=bd6592c37bb2,40087,1732486423713 2024-11-24T22:16:12,087 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732486556329.744bbec6934be304e01b009be760323d., hostname=bd6592c37bb2,40087,1732486423713, seqNum=2 , the old value is region=TestAcidGuarantees,,1732486556329.744bbec6934be304e01b009be760323d., hostname=bd6592c37bb2,40087,1732486423713, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=744bbec6934be304e01b009be760323d, server=bd6592c37bb2,40087,1732486423713 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T22:16:12,087 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40087 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=744bbec6934be304e01b009be760323d, server=bd6592c37bb2,40087,1732486423713 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-24T22:16:12,087 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732486556329.744bbec6934be304e01b009be760323d., hostname=bd6592c37bb2,40087,1732486423713, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=744bbec6934be304e01b009be760323d, server=bd6592c37bb2,40087,1732486423713 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T22:16:12,087 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732486556329.744bbec6934be304e01b009be760323d., hostname=bd6592c37bb2,40087,1732486423713, seqNum=2 because the exception is null or not the one we care about 2024-11-24T22:16:12,087 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40087 {}] ipc.CallRunner(138): callId: 53 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46816 deadline: 1732486632081, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=744bbec6934be304e01b009be760323d, server=bd6592c37bb2,40087,1732486423713 2024-11-24T22:16:12,088 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732486556329.744bbec6934be304e01b009be760323d., hostname=bd6592c37bb2,40087,1732486423713, seqNum=2 , the old value is region=TestAcidGuarantees,,1732486556329.744bbec6934be304e01b009be760323d., hostname=bd6592c37bb2,40087,1732486423713, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=744bbec6934be304e01b009be760323d, server=bd6592c37bb2,40087,1732486423713 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T22:16:12,088 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732486556329.744bbec6934be304e01b009be760323d., hostname=bd6592c37bb2,40087,1732486423713, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=744bbec6934be304e01b009be760323d, server=bd6592c37bb2,40087,1732486423713 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T22:16:12,088 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732486556329.744bbec6934be304e01b009be760323d., hostname=bd6592c37bb2,40087,1732486423713, seqNum=2 because the exception is null or not the one we care about 2024-11-24T22:16:12,106 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40087 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=744bbec6934be304e01b009be760323d, server=bd6592c37bb2,40087,1732486423713 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-24T22:16:12,106 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40087 {}] ipc.CallRunner(138): callId: 57 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46842 deadline: 1732486632100, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=744bbec6934be304e01b009be760323d, server=bd6592c37bb2,40087,1732486423713 2024-11-24T22:16:12,107 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732486556329.744bbec6934be304e01b009be760323d., hostname=bd6592c37bb2,40087,1732486423713, seqNum=2 , the old value is region=TestAcidGuarantees,,1732486556329.744bbec6934be304e01b009be760323d., hostname=bd6592c37bb2,40087,1732486423713, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=744bbec6934be304e01b009be760323d, server=bd6592c37bb2,40087,1732486423713 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T22:16:12,107 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732486556329.744bbec6934be304e01b009be760323d., hostname=bd6592c37bb2,40087,1732486423713, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=744bbec6934be304e01b009be760323d, server=bd6592c37bb2,40087,1732486423713 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T22:16:12,107 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732486556329.744bbec6934be304e01b009be760323d., hostname=bd6592c37bb2,40087,1732486423713, seqNum=2 because the exception is null or not the one we care about 2024-11-24T22:16:12,118 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40087 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=744bbec6934be304e01b009be760323d, server=bd6592c37bb2,40087,1732486423713 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-24T22:16:12,118 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40087 {}] ipc.CallRunner(138): callId: 136 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46792 deadline: 1732486632111, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=744bbec6934be304e01b009be760323d, server=bd6592c37bb2,40087,1732486423713 2024-11-24T22:16:12,118 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40087 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=744bbec6934be304e01b009be760323d, server=bd6592c37bb2,40087,1732486423713 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-24T22:16:12,118 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732486556329.744bbec6934be304e01b009be760323d., hostname=bd6592c37bb2,40087,1732486423713, seqNum=2 , the old value is region=TestAcidGuarantees,,1732486556329.744bbec6934be304e01b009be760323d., hostname=bd6592c37bb2,40087,1732486423713, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=744bbec6934be304e01b009be760323d, server=bd6592c37bb2,40087,1732486423713 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T22:16:12,118 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732486556329.744bbec6934be304e01b009be760323d., hostname=bd6592c37bb2,40087,1732486423713, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=744bbec6934be304e01b009be760323d, server=bd6592c37bb2,40087,1732486423713 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T22:16:12,118 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40087 {}] ipc.CallRunner(138): callId: 57 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46820 deadline: 1732486632111, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=744bbec6934be304e01b009be760323d, server=bd6592c37bb2,40087,1732486423713 2024-11-24T22:16:12,118 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732486556329.744bbec6934be304e01b009be760323d., hostname=bd6592c37bb2,40087,1732486423713, seqNum=2 because the exception is null or not the one we care about 2024-11-24T22:16:12,119 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732486556329.744bbec6934be304e01b009be760323d., hostname=bd6592c37bb2,40087,1732486423713, seqNum=2 , the old value is region=TestAcidGuarantees,,1732486556329.744bbec6934be304e01b009be760323d., hostname=bd6592c37bb2,40087,1732486423713, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=744bbec6934be304e01b009be760323d, server=bd6592c37bb2,40087,1732486423713 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T22:16:12,119 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732486556329.744bbec6934be304e01b009be760323d., hostname=bd6592c37bb2,40087,1732486423713, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=744bbec6934be304e01b009be760323d, server=bd6592c37bb2,40087,1732486423713 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T22:16:12,119 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732486556329.744bbec6934be304e01b009be760323d., hostname=bd6592c37bb2,40087,1732486423713, seqNum=2 because the exception is null or not the one we care about 2024-11-24T22:16:12,143 INFO [RS_FLUSH_OPERATIONS-regionserver/bd6592c37bb2:0-0 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=35.78 KB at sequenceid=255 (bloomFilter=true), to=hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/744bbec6934be304e01b009be760323d/.tmp/C/90db8ce5d3d54446be80c89ee834828f 2024-11-24T22:16:12,147 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bd6592c37bb2:0-0 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/744bbec6934be304e01b009be760323d/.tmp/A/6b1fe8f1a1474845bad917d19afeb88c as hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/744bbec6934be304e01b009be760323d/A/6b1fe8f1a1474845bad917d19afeb88c 2024-11-24T22:16:12,150 INFO [RS_FLUSH_OPERATIONS-regionserver/bd6592c37bb2:0-0 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/744bbec6934be304e01b009be760323d/A/6b1fe8f1a1474845bad917d19afeb88c, entries=150, sequenceid=255, filesize=11.9 K 2024-11-24T22:16:12,151 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bd6592c37bb2:0-0 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/744bbec6934be304e01b009be760323d/.tmp/B/58027f2e64c3466da9d9ec87e36f74e6 as hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/744bbec6934be304e01b009be760323d/B/58027f2e64c3466da9d9ec87e36f74e6 2024-11-24T22:16:12,153 INFO [RS_FLUSH_OPERATIONS-regionserver/bd6592c37bb2:0-0 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/744bbec6934be304e01b009be760323d/B/58027f2e64c3466da9d9ec87e36f74e6, entries=150, sequenceid=255, filesize=11.9 K 2024-11-24T22:16:12,154 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bd6592c37bb2:0-0 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/744bbec6934be304e01b009be760323d/.tmp/C/90db8ce5d3d54446be80c89ee834828f as hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/744bbec6934be304e01b009be760323d/C/90db8ce5d3d54446be80c89ee834828f 2024-11-24T22:16:12,157 INFO [RS_FLUSH_OPERATIONS-regionserver/bd6592c37bb2:0-0 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/744bbec6934be304e01b009be760323d/C/90db8ce5d3d54446be80c89ee834828f, entries=150, sequenceid=255, filesize=11.9 K 2024-11-24T22:16:12,158 INFO [RS_FLUSH_OPERATIONS-regionserver/bd6592c37bb2:0-0 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.HRegion(3140): Finished flush of dataSize ~107.34 KB/109920, heapSize ~319.45 KB/327120, currentSize=73.80 KB/75570 for 744bbec6934be304e01b009be760323d in 852ms, sequenceid=255, compaction requested=false 2024-11-24T22:16:12,158 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bd6592c37bb2:0-0 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.HRegion(2603): Flush status journal for 744bbec6934be304e01b009be760323d: 2024-11-24T22:16:12,158 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bd6592c37bb2:0-0 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732486556329.744bbec6934be304e01b009be760323d. 2024-11-24T22:16:12,158 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bd6592c37bb2:0-0 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=130 2024-11-24T22:16:12,159 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37701 {}] master.HMaster(4169): Remote procedure done, pid=130 2024-11-24T22:16:12,161 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=130, resume processing ppid=129 2024-11-24T22:16:12,161 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=130, ppid=129, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.0060 sec 2024-11-24T22:16:12,163 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=129, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=129, table=TestAcidGuarantees in 1.0110 sec 2024-11-24T22:16:12,291 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37701 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=129 2024-11-24T22:16:12,291 INFO [RPCClient-NioEventLoopGroup-4-10 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-24T22:16:12,292 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37701 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-24T22:16:12,293 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37701 {}] procedure2.ProcedureExecutor(1139): Stored pid=131, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=131, table=TestAcidGuarantees 2024-11-24T22:16:12,293 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37701 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=131 2024-11-24T22:16:12,293 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=131, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=131, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-24T22:16:12,294 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=131, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=131, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-24T22:16:12,294 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=132, ppid=131, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-24T22:16:12,401 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37701 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=131 2024-11-24T22:16:12,446 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=40087 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=132 2024-11-24T22:16:12,447 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bd6592c37bb2:0-1 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732486556329.744bbec6934be304e01b009be760323d. 2024-11-24T22:16:12,447 INFO [RS_FLUSH_OPERATIONS-regionserver/bd6592c37bb2:0-1 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.HRegion(2902): Flushing 744bbec6934be304e01b009be760323d 3/3 column families, dataSize=73.80 KB heapSize=219.89 KB 2024-11-24T22:16:12,447 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bd6592c37bb2:0-1 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 744bbec6934be304e01b009be760323d, store=A 2024-11-24T22:16:12,447 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bd6592c37bb2:0-1 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-24T22:16:12,447 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bd6592c37bb2:0-1 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 744bbec6934be304e01b009be760323d, store=B 2024-11-24T22:16:12,447 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bd6592c37bb2:0-1 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-24T22:16:12,447 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bd6592c37bb2:0-1 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 744bbec6934be304e01b009be760323d, store=C 2024-11-24T22:16:12,447 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bd6592c37bb2:0-1 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-24T22:16:12,452 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bd6592c37bb2:0-1 {event_type=RS_FLUSH_REGIONS, pid=132}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/744bbec6934be304e01b009be760323d/.tmp/A/021a9989574341f2a117574e6756a07a is 50, key is test_row_0/A:col10/1732486571445/Put/seqid=0 2024-11-24T22:16:12,455 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45929 is added to blk_1073742335_1511 (size=12301) 2024-11-24T22:16:12,460 INFO [RS_FLUSH_OPERATIONS-regionserver/bd6592c37bb2:0-1 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=24.60 KB at sequenceid=271 (bloomFilter=true), to=hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/744bbec6934be304e01b009be760323d/.tmp/A/021a9989574341f2a117574e6756a07a 2024-11-24T22:16:12,475 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bd6592c37bb2:0-1 {event_type=RS_FLUSH_REGIONS, pid=132}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/744bbec6934be304e01b009be760323d/.tmp/B/0ee2bcf3005f4e8197e016dfb0809d3e is 50, key is test_row_0/B:col10/1732486571445/Put/seqid=0 2024-11-24T22:16:12,489 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45929 is added to blk_1073742336_1512 (size=12301) 2024-11-24T22:16:12,490 INFO [RS_FLUSH_OPERATIONS-regionserver/bd6592c37bb2:0-1 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=24.60 KB at sequenceid=271 (bloomFilter=true), to=hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/744bbec6934be304e01b009be760323d/.tmp/B/0ee2bcf3005f4e8197e016dfb0809d3e 2024-11-24T22:16:12,503 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bd6592c37bb2:0-1 {event_type=RS_FLUSH_REGIONS, pid=132}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/744bbec6934be304e01b009be760323d/.tmp/C/76252ddb4dd84d358abcfb118a4bb6b4 is 50, key is test_row_0/C:col10/1732486571445/Put/seqid=0 2024-11-24T22:16:12,507 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45929 is added to blk_1073742337_1513 (size=12301) 2024-11-24T22:16:12,509 INFO [RS_FLUSH_OPERATIONS-regionserver/bd6592c37bb2:0-1 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=24.60 KB at sequenceid=271 (bloomFilter=true), to=hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/744bbec6934be304e01b009be760323d/.tmp/C/76252ddb4dd84d358abcfb118a4bb6b4 2024-11-24T22:16:12,512 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bd6592c37bb2:0-1 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/744bbec6934be304e01b009be760323d/.tmp/A/021a9989574341f2a117574e6756a07a as hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/744bbec6934be304e01b009be760323d/A/021a9989574341f2a117574e6756a07a 2024-11-24T22:16:12,515 INFO [RS_FLUSH_OPERATIONS-regionserver/bd6592c37bb2:0-1 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/744bbec6934be304e01b009be760323d/A/021a9989574341f2a117574e6756a07a, entries=150, sequenceid=271, filesize=12.0 K 2024-11-24T22:16:12,516 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bd6592c37bb2:0-1 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/744bbec6934be304e01b009be760323d/.tmp/B/0ee2bcf3005f4e8197e016dfb0809d3e as hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/744bbec6934be304e01b009be760323d/B/0ee2bcf3005f4e8197e016dfb0809d3e 2024-11-24T22:16:12,519 INFO [RS_FLUSH_OPERATIONS-regionserver/bd6592c37bb2:0-1 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/744bbec6934be304e01b009be760323d/B/0ee2bcf3005f4e8197e016dfb0809d3e, entries=150, sequenceid=271, filesize=12.0 K 2024-11-24T22:16:12,519 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bd6592c37bb2:0-1 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/744bbec6934be304e01b009be760323d/.tmp/C/76252ddb4dd84d358abcfb118a4bb6b4 as hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/744bbec6934be304e01b009be760323d/C/76252ddb4dd84d358abcfb118a4bb6b4 2024-11-24T22:16:12,523 INFO [RS_FLUSH_OPERATIONS-regionserver/bd6592c37bb2:0-1 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/744bbec6934be304e01b009be760323d/C/76252ddb4dd84d358abcfb118a4bb6b4, entries=150, sequenceid=271, filesize=12.0 K 2024-11-24T22:16:12,523 INFO [RS_FLUSH_OPERATIONS-regionserver/bd6592c37bb2:0-1 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.HRegion(3140): Finished flush of dataSize ~73.80 KB/75570, heapSize ~219.84 KB/225120, currentSize=0 B/0 for 744bbec6934be304e01b009be760323d in 76ms, sequenceid=271, compaction requested=true 2024-11-24T22:16:12,523 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bd6592c37bb2:0-1 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.HRegion(2603): Flush status journal for 744bbec6934be304e01b009be760323d: 2024-11-24T22:16:12,523 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bd6592c37bb2:0-1 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732486556329.744bbec6934be304e01b009be760323d. 2024-11-24T22:16:12,524 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bd6592c37bb2:0-1 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=132 2024-11-24T22:16:12,526 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37701 {}] master.HMaster(4169): Remote procedure done, pid=132 2024-11-24T22:16:12,528 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=132, resume processing ppid=131 2024-11-24T22:16:12,528 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=132, ppid=131, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 233 msec 2024-11-24T22:16:12,530 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=131, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=131, table=TestAcidGuarantees in 237 msec 2024-11-24T22:16:12,611 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37701 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=131 2024-11-24T22:16:12,611 INFO [RPCClient-NioEventLoopGroup-4-10 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-24T22:16:12,612 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37701 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-24T22:16:12,613 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37701 {}] procedure2.ProcedureExecutor(1139): Stored pid=133, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=133, table=TestAcidGuarantees 2024-11-24T22:16:12,614 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37701 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=133 2024-11-24T22:16:12,614 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=133, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=133, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-24T22:16:12,615 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=133, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=133, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-24T22:16:12,615 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=134, ppid=133, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-24T22:16:12,627 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40087 {}] regionserver.HRegion(8855): Flush requested on 744bbec6934be304e01b009be760323d 2024-11-24T22:16:12,627 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 744bbec6934be304e01b009be760323d 3/3 column families, dataSize=46.96 KB heapSize=140.20 KB 2024-11-24T22:16:12,629 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 744bbec6934be304e01b009be760323d, store=A 2024-11-24T22:16:12,629 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-24T22:16:12,630 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 744bbec6934be304e01b009be760323d, store=B 2024-11-24T22:16:12,630 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-24T22:16:12,630 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 744bbec6934be304e01b009be760323d, store=C 2024-11-24T22:16:12,630 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-24T22:16:12,646 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/744bbec6934be304e01b009be760323d/.tmp/A/5a670e57236e459fb71bd3c4bbdd6550 is 50, key is test_row_0/A:col10/1732486572623/Put/seqid=0 2024-11-24T22:16:12,670 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45929 is added to blk_1073742338_1514 (size=14741) 2024-11-24T22:16:12,671 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=282 (bloomFilter=true), to=hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/744bbec6934be304e01b009be760323d/.tmp/A/5a670e57236e459fb71bd3c4bbdd6550 2024-11-24T22:16:12,689 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/744bbec6934be304e01b009be760323d/.tmp/B/2f3eb3c994b34515add661fedd73ef59 is 50, key is test_row_0/B:col10/1732486572623/Put/seqid=0 2024-11-24T22:16:12,706 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40087 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=744bbec6934be304e01b009be760323d, server=bd6592c37bb2,40087,1732486423713 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-24T22:16:12,707 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40087 {}] ipc.CallRunner(138): callId: 62 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46842 deadline: 1732486632689, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=744bbec6934be304e01b009be760323d, server=bd6592c37bb2,40087,1732486423713 2024-11-24T22:16:12,707 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/bd6592c37bb2:0-8 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=309, cellSize=14145, totalHeapSize=32376, min timestamp=1732486572677, max timestamp=1732486572704Num uniques 859; 2024-11-24T22:16:12,707 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/bd6592c37bb2:0-8 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=14145, suffix heap size=117064,new segment heap size=32376 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=309 2024-11-24T22:16:12,708 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/bd6592c37bb2:0-6 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=309, cellSize=14145, totalHeapSize=32376, min timestamp=1732486572677, max timestamp=1732486572704Num uniques 859; 2024-11-24T22:16:12,708 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/bd6592c37bb2:0-6 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=14145, suffix heap size=117064,new segment heap size=32376 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=309 2024-11-24T22:16:12,709 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732486556329.744bbec6934be304e01b009be760323d., hostname=bd6592c37bb2,40087,1732486423713, seqNum=2 , the old value is region=TestAcidGuarantees,,1732486556329.744bbec6934be304e01b009be760323d., hostname=bd6592c37bb2,40087,1732486423713, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=744bbec6934be304e01b009be760323d, server=bd6592c37bb2,40087,1732486423713 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T22:16:12,709 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732486556329.744bbec6934be304e01b009be760323d., hostname=bd6592c37bb2,40087,1732486423713, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=744bbec6934be304e01b009be760323d, server=bd6592c37bb2,40087,1732486423713 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T22:16:12,709 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732486556329.744bbec6934be304e01b009be760323d., hostname=bd6592c37bb2,40087,1732486423713, seqNum=2 because the exception is null or not the one we care about 2024-11-24T22:16:12,711 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45929 is added to blk_1073742339_1515 (size=12301) 2024-11-24T22:16:12,713 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=282 (bloomFilter=true), to=hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/744bbec6934be304e01b009be760323d/.tmp/B/2f3eb3c994b34515add661fedd73ef59 2024-11-24T22:16:12,721 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37701 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=133 2024-11-24T22:16:12,733 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/744bbec6934be304e01b009be760323d/.tmp/C/2583f4800d584c48b0cf73a317359c00 is 50, key is test_row_0/C:col10/1732486572623/Put/seqid=0 2024-11-24T22:16:12,750 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45929 is added to blk_1073742340_1516 (size=12301) 2024-11-24T22:16:12,762 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40087 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=744bbec6934be304e01b009be760323d, server=bd6592c37bb2,40087,1732486423713 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-24T22:16:12,763 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40087 {}] ipc.CallRunner(138): callId: 142 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46792 deadline: 1732486632740, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=744bbec6934be304e01b009be760323d, server=bd6592c37bb2,40087,1732486423713 2024-11-24T22:16:12,764 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732486556329.744bbec6934be304e01b009be760323d., hostname=bd6592c37bb2,40087,1732486423713, seqNum=2 , the old value is region=TestAcidGuarantees,,1732486556329.744bbec6934be304e01b009be760323d., hostname=bd6592c37bb2,40087,1732486423713, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=744bbec6934be304e01b009be760323d, server=bd6592c37bb2,40087,1732486423713 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T22:16:12,764 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732486556329.744bbec6934be304e01b009be760323d., hostname=bd6592c37bb2,40087,1732486423713, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=744bbec6934be304e01b009be760323d, server=bd6592c37bb2,40087,1732486423713 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T22:16:12,764 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732486556329.744bbec6934be304e01b009be760323d., hostname=bd6592c37bb2,40087,1732486423713, seqNum=2 because the exception is null or not the one we care about 2024-11-24T22:16:12,768 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=40087 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=134 2024-11-24T22:16:12,768 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bd6592c37bb2:0-2 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732486556329.744bbec6934be304e01b009be760323d. 2024-11-24T22:16:12,769 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bd6592c37bb2:0-2 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732486556329.744bbec6934be304e01b009be760323d. as already flushing 2024-11-24T22:16:12,769 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bd6592c37bb2:0-2 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732486556329.744bbec6934be304e01b009be760323d. 2024-11-24T22:16:12,769 ERROR [RS_FLUSH_OPERATIONS-regionserver/bd6592c37bb2:0-2 {event_type=RS_FLUSH_REGIONS, pid=134}] handler.RSProcedureHandler(58): pid=134 java.io.IOException: Unable to complete flush {ENCODED => 744bbec6934be304e01b009be760323d, NAME => 'TestAcidGuarantees,,1732486556329.744bbec6934be304e01b009be760323d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T22:16:12,769 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bd6592c37bb2:0-2 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=134 java.io.IOException: Unable to complete flush {ENCODED => 744bbec6934be304e01b009be760323d, NAME => 'TestAcidGuarantees,,1732486556329.744bbec6934be304e01b009be760323d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T22:16:12,770 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37701 {}] master.HMaster(4177): Remote procedure failed, pid=134 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 744bbec6934be304e01b009be760323d, NAME => 'TestAcidGuarantees,,1732486556329.744bbec6934be304e01b009be760323d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 744bbec6934be304e01b009be760323d, NAME => 'TestAcidGuarantees,,1732486556329.744bbec6934be304e01b009be760323d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T22:16:12,773 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40087 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=744bbec6934be304e01b009be760323d, server=bd6592c37bb2,40087,1732486423713 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-24T22:16:12,773 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40087 {}] ipc.CallRunner(138): callId: 64 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46820 deadline: 1732486632752, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=744bbec6934be304e01b009be760323d, server=bd6592c37bb2,40087,1732486423713 2024-11-24T22:16:12,776 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732486556329.744bbec6934be304e01b009be760323d., hostname=bd6592c37bb2,40087,1732486423713, seqNum=2 , the old value is region=TestAcidGuarantees,,1732486556329.744bbec6934be304e01b009be760323d., hostname=bd6592c37bb2,40087,1732486423713, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=744bbec6934be304e01b009be760323d, server=bd6592c37bb2,40087,1732486423713 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T22:16:12,776 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732486556329.744bbec6934be304e01b009be760323d., hostname=bd6592c37bb2,40087,1732486423713, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=744bbec6934be304e01b009be760323d, server=bd6592c37bb2,40087,1732486423713 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T22:16:12,776 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732486556329.744bbec6934be304e01b009be760323d., hostname=bd6592c37bb2,40087,1732486423713, seqNum=2 because the exception is null or not the one we care about 2024-11-24T22:16:12,776 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40087 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=744bbec6934be304e01b009be760323d, server=bd6592c37bb2,40087,1732486423713 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-24T22:16:12,776 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40087 {}] ipc.CallRunner(138): callId: 72 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46824 deadline: 1732486632754, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=744bbec6934be304e01b009be760323d, server=bd6592c37bb2,40087,1732486423713 2024-11-24T22:16:12,777 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732486556329.744bbec6934be304e01b009be760323d., hostname=bd6592c37bb2,40087,1732486423713, seqNum=2 , the old value is region=TestAcidGuarantees,,1732486556329.744bbec6934be304e01b009be760323d., hostname=bd6592c37bb2,40087,1732486423713, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=744bbec6934be304e01b009be760323d, server=bd6592c37bb2,40087,1732486423713 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T22:16:12,777 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732486556329.744bbec6934be304e01b009be760323d., hostname=bd6592c37bb2,40087,1732486423713, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=744bbec6934be304e01b009be760323d, server=bd6592c37bb2,40087,1732486423713 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T22:16:12,777 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40087 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=744bbec6934be304e01b009be760323d, server=bd6592c37bb2,40087,1732486423713 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-24T22:16:12,777 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732486556329.744bbec6934be304e01b009be760323d., hostname=bd6592c37bb2,40087,1732486423713, seqNum=2 because the exception is null or not the one we care about 2024-11-24T22:16:12,777 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40087 {}] ipc.CallRunner(138): callId: 63 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46816 deadline: 1732486632755, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=744bbec6934be304e01b009be760323d, server=bd6592c37bb2,40087,1732486423713 2024-11-24T22:16:12,777 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732486556329.744bbec6934be304e01b009be760323d., hostname=bd6592c37bb2,40087,1732486423713, seqNum=2 , the old value is region=TestAcidGuarantees,,1732486556329.744bbec6934be304e01b009be760323d., hostname=bd6592c37bb2,40087,1732486423713, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=744bbec6934be304e01b009be760323d, server=bd6592c37bb2,40087,1732486423713 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T22:16:12,777 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732486556329.744bbec6934be304e01b009be760323d., hostname=bd6592c37bb2,40087,1732486423713, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=744bbec6934be304e01b009be760323d, server=bd6592c37bb2,40087,1732486423713 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T22:16:12,777 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732486556329.744bbec6934be304e01b009be760323d., hostname=bd6592c37bb2,40087,1732486423713, seqNum=2 because the exception is null or not the one we care about 2024-11-24T22:16:12,825 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40087 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=744bbec6934be304e01b009be760323d, server=bd6592c37bb2,40087,1732486423713 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-24T22:16:12,825 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40087 {}] ipc.CallRunner(138): callId: 63 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46842 deadline: 1732486632821, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=744bbec6934be304e01b009be760323d, server=bd6592c37bb2,40087,1732486423713 2024-11-24T22:16:12,825 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732486556329.744bbec6934be304e01b009be760323d., hostname=bd6592c37bb2,40087,1732486423713, seqNum=2 , the old value is region=TestAcidGuarantees,,1732486556329.744bbec6934be304e01b009be760323d., hostname=bd6592c37bb2,40087,1732486423713, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=744bbec6934be304e01b009be760323d, server=bd6592c37bb2,40087,1732486423713 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T22:16:12,826 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732486556329.744bbec6934be304e01b009be760323d., hostname=bd6592c37bb2,40087,1732486423713, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=744bbec6934be304e01b009be760323d, server=bd6592c37bb2,40087,1732486423713 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T22:16:12,826 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732486556329.744bbec6934be304e01b009be760323d., hostname=bd6592c37bb2,40087,1732486423713, seqNum=2 because the exception is null or not the one we care about 2024-11-24T22:16:12,885 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40087 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=744bbec6934be304e01b009be760323d, server=bd6592c37bb2,40087,1732486423713 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-24T22:16:12,885 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40087 {}] ipc.CallRunner(138): callId: 143 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46792 deadline: 1732486632870, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=744bbec6934be304e01b009be760323d, server=bd6592c37bb2,40087,1732486423713 2024-11-24T22:16:12,886 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732486556329.744bbec6934be304e01b009be760323d., hostname=bd6592c37bb2,40087,1732486423713, seqNum=2 , the old value is region=TestAcidGuarantees,,1732486556329.744bbec6934be304e01b009be760323d., hostname=bd6592c37bb2,40087,1732486423713, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=744bbec6934be304e01b009be760323d, server=bd6592c37bb2,40087,1732486423713 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T22:16:12,886 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732486556329.744bbec6934be304e01b009be760323d., hostname=bd6592c37bb2,40087,1732486423713, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=744bbec6934be304e01b009be760323d, server=bd6592c37bb2,40087,1732486423713 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T22:16:12,886 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732486556329.744bbec6934be304e01b009be760323d., hostname=bd6592c37bb2,40087,1732486423713, seqNum=2 because the exception is null or not the one we care about 2024-11-24T22:16:12,895 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40087 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=744bbec6934be304e01b009be760323d, server=bd6592c37bb2,40087,1732486423713 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-24T22:16:12,895 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40087 {}] ipc.CallRunner(138): callId: 65 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46820 deadline: 1732486632881, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=744bbec6934be304e01b009be760323d, server=bd6592c37bb2,40087,1732486423713 2024-11-24T22:16:12,895 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732486556329.744bbec6934be304e01b009be760323d., hostname=bd6592c37bb2,40087,1732486423713, seqNum=2 , the old value is region=TestAcidGuarantees,,1732486556329.744bbec6934be304e01b009be760323d., hostname=bd6592c37bb2,40087,1732486423713, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=744bbec6934be304e01b009be760323d, server=bd6592c37bb2,40087,1732486423713 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T22:16:12,895 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40087 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=744bbec6934be304e01b009be760323d, server=bd6592c37bb2,40087,1732486423713 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-24T22:16:12,895 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732486556329.744bbec6934be304e01b009be760323d., hostname=bd6592c37bb2,40087,1732486423713, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=744bbec6934be304e01b009be760323d, server=bd6592c37bb2,40087,1732486423713 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T22:16:12,895 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732486556329.744bbec6934be304e01b009be760323d., hostname=bd6592c37bb2,40087,1732486423713, seqNum=2 because the exception is null or not the one we care about 2024-11-24T22:16:12,895 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40087 {}] ipc.CallRunner(138): callId: 64 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46816 deadline: 1732486632881, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=744bbec6934be304e01b009be760323d, server=bd6592c37bb2,40087,1732486423713 2024-11-24T22:16:12,896 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40087 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=744bbec6934be304e01b009be760323d, server=bd6592c37bb2,40087,1732486423713 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-24T22:16:12,896 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40087 {}] ipc.CallRunner(138): callId: 73 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46824 deadline: 1732486632881, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=744bbec6934be304e01b009be760323d, server=bd6592c37bb2,40087,1732486423713 2024-11-24T22:16:12,897 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732486556329.744bbec6934be304e01b009be760323d., hostname=bd6592c37bb2,40087,1732486423713, seqNum=2 , the old value is region=TestAcidGuarantees,,1732486556329.744bbec6934be304e01b009be760323d., hostname=bd6592c37bb2,40087,1732486423713, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=744bbec6934be304e01b009be760323d, server=bd6592c37bb2,40087,1732486423713 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T22:16:12,897 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732486556329.744bbec6934be304e01b009be760323d., hostname=bd6592c37bb2,40087,1732486423713, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=744bbec6934be304e01b009be760323d, server=bd6592c37bb2,40087,1732486423713 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T22:16:12,897 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732486556329.744bbec6934be304e01b009be760323d., hostname=bd6592c37bb2,40087,1732486423713, seqNum=2 because the exception is null or not the one we care about 2024-11-24T22:16:12,900 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732486556329.744bbec6934be304e01b009be760323d., hostname=bd6592c37bb2,40087,1732486423713, seqNum=2 , the old value is region=TestAcidGuarantees,,1732486556329.744bbec6934be304e01b009be760323d., hostname=bd6592c37bb2,40087,1732486423713, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=744bbec6934be304e01b009be760323d, server=bd6592c37bb2,40087,1732486423713 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T22:16:12,900 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732486556329.744bbec6934be304e01b009be760323d., hostname=bd6592c37bb2,40087,1732486423713, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=744bbec6934be304e01b009be760323d, server=bd6592c37bb2,40087,1732486423713 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T22:16:12,900 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732486556329.744bbec6934be304e01b009be760323d., hostname=bd6592c37bb2,40087,1732486423713, seqNum=2 because the exception is null or not the one we care about 2024-11-24T22:16:12,922 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=40087 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=134 2024-11-24T22:16:12,922 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bd6592c37bb2:0-0 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732486556329.744bbec6934be304e01b009be760323d. 2024-11-24T22:16:12,922 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bd6592c37bb2:0-0 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732486556329.744bbec6934be304e01b009be760323d. as already flushing 2024-11-24T22:16:12,922 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bd6592c37bb2:0-0 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732486556329.744bbec6934be304e01b009be760323d. 2024-11-24T22:16:12,922 ERROR [RS_FLUSH_OPERATIONS-regionserver/bd6592c37bb2:0-0 {event_type=RS_FLUSH_REGIONS, pid=134}] handler.RSProcedureHandler(58): pid=134 java.io.IOException: Unable to complete flush {ENCODED => 744bbec6934be304e01b009be760323d, NAME => 'TestAcidGuarantees,,1732486556329.744bbec6934be304e01b009be760323d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T22:16:12,922 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bd6592c37bb2:0-0 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=134 java.io.IOException: Unable to complete flush {ENCODED => 744bbec6934be304e01b009be760323d, NAME => 'TestAcidGuarantees,,1732486556329.744bbec6934be304e01b009be760323d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T22:16:12,923 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37701 {}] master.HMaster(4177): Remote procedure failed, pid=134 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 744bbec6934be304e01b009be760323d, NAME => 'TestAcidGuarantees,,1732486556329.744bbec6934be304e01b009be760323d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 744bbec6934be304e01b009be760323d, NAME => 'TestAcidGuarantees,,1732486556329.744bbec6934be304e01b009be760323d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T22:16:12,930 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37701 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=133 2024-11-24T22:16:13,042 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40087 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=744bbec6934be304e01b009be760323d, server=bd6592c37bb2,40087,1732486423713 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-24T22:16:13,042 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40087 {}] ipc.CallRunner(138): callId: 64 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46842 deadline: 1732486633031, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=744bbec6934be304e01b009be760323d, server=bd6592c37bb2,40087,1732486423713 2024-11-24T22:16:13,043 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732486556329.744bbec6934be304e01b009be760323d., hostname=bd6592c37bb2,40087,1732486423713, seqNum=2 , the old value is region=TestAcidGuarantees,,1732486556329.744bbec6934be304e01b009be760323d., hostname=bd6592c37bb2,40087,1732486423713, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=744bbec6934be304e01b009be760323d, server=bd6592c37bb2,40087,1732486423713 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T22:16:13,043 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732486556329.744bbec6934be304e01b009be760323d., hostname=bd6592c37bb2,40087,1732486423713, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=744bbec6934be304e01b009be760323d, server=bd6592c37bb2,40087,1732486423713 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T22:16:13,043 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732486556329.744bbec6934be304e01b009be760323d., hostname=bd6592c37bb2,40087,1732486423713, seqNum=2 because the exception is null or not the one we care about 2024-11-24T22:16:13,075 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=40087 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=134 2024-11-24T22:16:13,075 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bd6592c37bb2:0-1 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732486556329.744bbec6934be304e01b009be760323d. 2024-11-24T22:16:13,075 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bd6592c37bb2:0-1 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732486556329.744bbec6934be304e01b009be760323d. as already flushing 2024-11-24T22:16:13,075 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bd6592c37bb2:0-1 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732486556329.744bbec6934be304e01b009be760323d. 2024-11-24T22:16:13,075 ERROR [RS_FLUSH_OPERATIONS-regionserver/bd6592c37bb2:0-1 {event_type=RS_FLUSH_REGIONS, pid=134}] handler.RSProcedureHandler(58): pid=134 java.io.IOException: Unable to complete flush {ENCODED => 744bbec6934be304e01b009be760323d, NAME => 'TestAcidGuarantees,,1732486556329.744bbec6934be304e01b009be760323d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T22:16:13,075 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bd6592c37bb2:0-1 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=134 java.io.IOException: Unable to complete flush {ENCODED => 744bbec6934be304e01b009be760323d, NAME => 'TestAcidGuarantees,,1732486556329.744bbec6934be304e01b009be760323d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T22:16:13,076 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37701 {}] master.HMaster(4177): Remote procedure failed, pid=134 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 744bbec6934be304e01b009be760323d, NAME => 'TestAcidGuarantees,,1732486556329.744bbec6934be304e01b009be760323d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 744bbec6934be304e01b009be760323d, NAME => 'TestAcidGuarantees,,1732486556329.744bbec6934be304e01b009be760323d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T22:16:13,096 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40087 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=744bbec6934be304e01b009be760323d, server=bd6592c37bb2,40087,1732486423713 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-24T22:16:13,096 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40087 {}] ipc.CallRunner(138): callId: 144 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46792 deadline: 1732486633091, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=744bbec6934be304e01b009be760323d, server=bd6592c37bb2,40087,1732486423713 2024-11-24T22:16:13,097 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732486556329.744bbec6934be304e01b009be760323d., hostname=bd6592c37bb2,40087,1732486423713, seqNum=2 , the old value is region=TestAcidGuarantees,,1732486556329.744bbec6934be304e01b009be760323d., hostname=bd6592c37bb2,40087,1732486423713, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=744bbec6934be304e01b009be760323d, server=bd6592c37bb2,40087,1732486423713 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T22:16:13,097 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732486556329.744bbec6934be304e01b009be760323d., hostname=bd6592c37bb2,40087,1732486423713, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=744bbec6934be304e01b009be760323d, server=bd6592c37bb2,40087,1732486423713 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T22:16:13,097 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732486556329.744bbec6934be304e01b009be760323d., hostname=bd6592c37bb2,40087,1732486423713, seqNum=2 because the exception is null or not the one we care about 2024-11-24T22:16:13,115 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40087 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=744bbec6934be304e01b009be760323d, server=bd6592c37bb2,40087,1732486423713 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-24T22:16:13,115 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40087 {}] ipc.CallRunner(138): callId: 66 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46820 deadline: 1732486633101, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=744bbec6934be304e01b009be760323d, server=bd6592c37bb2,40087,1732486423713 2024-11-24T22:16:13,115 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40087 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=744bbec6934be304e01b009be760323d, server=bd6592c37bb2,40087,1732486423713 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-24T22:16:13,115 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40087 {}] ipc.CallRunner(138): callId: 74 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46824 deadline: 1732486633101, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=744bbec6934be304e01b009be760323d, server=bd6592c37bb2,40087,1732486423713 2024-11-24T22:16:13,116 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732486556329.744bbec6934be304e01b009be760323d., hostname=bd6592c37bb2,40087,1732486423713, seqNum=2 , the old value is region=TestAcidGuarantees,,1732486556329.744bbec6934be304e01b009be760323d., hostname=bd6592c37bb2,40087,1732486423713, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=744bbec6934be304e01b009be760323d, server=bd6592c37bb2,40087,1732486423713 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T22:16:13,116 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732486556329.744bbec6934be304e01b009be760323d., hostname=bd6592c37bb2,40087,1732486423713, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=744bbec6934be304e01b009be760323d, server=bd6592c37bb2,40087,1732486423713 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T22:16:13,116 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732486556329.744bbec6934be304e01b009be760323d., hostname=bd6592c37bb2,40087,1732486423713, seqNum=2 because the exception is null or not the one we care about 2024-11-24T22:16:13,116 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40087 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=744bbec6934be304e01b009be760323d, server=bd6592c37bb2,40087,1732486423713 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-24T22:16:13,116 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40087 {}] ipc.CallRunner(138): callId: 65 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46816 deadline: 1732486633110, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=744bbec6934be304e01b009be760323d, server=bd6592c37bb2,40087,1732486423713 2024-11-24T22:16:13,116 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732486556329.744bbec6934be304e01b009be760323d., hostname=bd6592c37bb2,40087,1732486423713, seqNum=2 , the old value is region=TestAcidGuarantees,,1732486556329.744bbec6934be304e01b009be760323d., hostname=bd6592c37bb2,40087,1732486423713, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=744bbec6934be304e01b009be760323d, server=bd6592c37bb2,40087,1732486423713 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T22:16:13,116 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732486556329.744bbec6934be304e01b009be760323d., hostname=bd6592c37bb2,40087,1732486423713, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=744bbec6934be304e01b009be760323d, server=bd6592c37bb2,40087,1732486423713 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T22:16:13,116 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732486556329.744bbec6934be304e01b009be760323d., hostname=bd6592c37bb2,40087,1732486423713, seqNum=2 because the exception is null or not the one we care about 2024-11-24T22:16:13,116 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732486556329.744bbec6934be304e01b009be760323d., hostname=bd6592c37bb2,40087,1732486423713, seqNum=2 , the old value is region=TestAcidGuarantees,,1732486556329.744bbec6934be304e01b009be760323d., hostname=bd6592c37bb2,40087,1732486423713, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=744bbec6934be304e01b009be760323d, server=bd6592c37bb2,40087,1732486423713 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T22:16:13,116 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732486556329.744bbec6934be304e01b009be760323d., hostname=bd6592c37bb2,40087,1732486423713, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=744bbec6934be304e01b009be760323d, server=bd6592c37bb2,40087,1732486423713 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T22:16:13,116 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732486556329.744bbec6934be304e01b009be760323d., hostname=bd6592c37bb2,40087,1732486423713, seqNum=2 because the exception is null or not the one we care about 2024-11-24T22:16:13,149 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=282 (bloomFilter=true), to=hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/744bbec6934be304e01b009be760323d/.tmp/C/2583f4800d584c48b0cf73a317359c00 2024-11-24T22:16:13,153 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/744bbec6934be304e01b009be760323d/.tmp/A/5a670e57236e459fb71bd3c4bbdd6550 as hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/744bbec6934be304e01b009be760323d/A/5a670e57236e459fb71bd3c4bbdd6550 2024-11-24T22:16:13,157 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/744bbec6934be304e01b009be760323d/A/5a670e57236e459fb71bd3c4bbdd6550, entries=200, sequenceid=282, filesize=14.4 K 2024-11-24T22:16:13,157 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/744bbec6934be304e01b009be760323d/.tmp/B/2f3eb3c994b34515add661fedd73ef59 as hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/744bbec6934be304e01b009be760323d/B/2f3eb3c994b34515add661fedd73ef59 2024-11-24T22:16:13,161 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/744bbec6934be304e01b009be760323d/B/2f3eb3c994b34515add661fedd73ef59, entries=150, sequenceid=282, filesize=12.0 K 2024-11-24T22:16:13,162 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/744bbec6934be304e01b009be760323d/.tmp/C/2583f4800d584c48b0cf73a317359c00 as hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/744bbec6934be304e01b009be760323d/C/2583f4800d584c48b0cf73a317359c00 2024-11-24T22:16:13,165 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/744bbec6934be304e01b009be760323d/C/2583f4800d584c48b0cf73a317359c00, entries=150, sequenceid=282, filesize=12.0 K 2024-11-24T22:16:13,165 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~53.67 KB/54960, heapSize ~160.08 KB/163920, currentSize=125.23 KB/128240 for 744bbec6934be304e01b009be760323d in 538ms, sequenceid=282, compaction requested=true 2024-11-24T22:16:13,166 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 744bbec6934be304e01b009be760323d: 2024-11-24T22:16:13,166 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 744bbec6934be304e01b009be760323d:A, priority=-2147483648, current under compaction store size is 1 2024-11-24T22:16:13,166 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-24T22:16:13,166 DEBUG [RS:0;bd6592c37bb2:40087-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-24T22:16:13,166 DEBUG [RS:0;bd6592c37bb2:40087-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-24T22:16:13,166 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 744bbec6934be304e01b009be760323d:B, priority=-2147483648, current under compaction store size is 2 2024-11-24T22:16:13,166 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-24T22:16:13,166 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 744bbec6934be304e01b009be760323d:C, priority=-2147483648, current under compaction store size is 3 2024-11-24T22:16:13,166 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-24T22:16:13,167 DEBUG [RS:0;bd6592c37bb2:40087-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 51924 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-24T22:16:13,167 DEBUG [RS:0;bd6592c37bb2:40087-shortCompactions-0 {}] regionserver.HStore(1541): 744bbec6934be304e01b009be760323d/A is initiating minor compaction (all files) 2024-11-24T22:16:13,167 INFO [RS:0;bd6592c37bb2:40087-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 744bbec6934be304e01b009be760323d/A in TestAcidGuarantees,,1732486556329.744bbec6934be304e01b009be760323d. 2024-11-24T22:16:13,167 DEBUG [RS:0;bd6592c37bb2:40087-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 49484 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-24T22:16:13,167 DEBUG [RS:0;bd6592c37bb2:40087-longCompactions-0 {}] regionserver.HStore(1541): 744bbec6934be304e01b009be760323d/B is initiating minor compaction (all files) 2024-11-24T22:16:13,167 INFO [RS:0;bd6592c37bb2:40087-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/744bbec6934be304e01b009be760323d/A/5ad98303c5714401b416c2ba4baaaff5, hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/744bbec6934be304e01b009be760323d/A/6b1fe8f1a1474845bad917d19afeb88c, hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/744bbec6934be304e01b009be760323d/A/021a9989574341f2a117574e6756a07a, hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/744bbec6934be304e01b009be760323d/A/5a670e57236e459fb71bd3c4bbdd6550] into tmpdir=hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/744bbec6934be304e01b009be760323d/.tmp, totalSize=50.7 K 2024-11-24T22:16:13,167 INFO [RS:0;bd6592c37bb2:40087-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 744bbec6934be304e01b009be760323d/B in TestAcidGuarantees,,1732486556329.744bbec6934be304e01b009be760323d. 2024-11-24T22:16:13,167 INFO [RS:0;bd6592c37bb2:40087-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/744bbec6934be304e01b009be760323d/B/dd7740f4d1404c9a8367cc54fe2fbdd3, hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/744bbec6934be304e01b009be760323d/B/58027f2e64c3466da9d9ec87e36f74e6, hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/744bbec6934be304e01b009be760323d/B/0ee2bcf3005f4e8197e016dfb0809d3e, hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/744bbec6934be304e01b009be760323d/B/2f3eb3c994b34515add661fedd73ef59] into tmpdir=hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/744bbec6934be304e01b009be760323d/.tmp, totalSize=48.3 K 2024-11-24T22:16:13,167 DEBUG [RS:0;bd6592c37bb2:40087-shortCompactions-0 {}] compactions.Compactor(225): Compacting 5ad98303c5714401b416c2ba4baaaff5, keycount=150, bloomtype=ROW, size=12.4 K, encoding=NONE, compression=NONE, seqNum=235, earliestPutTs=1732486570600 2024-11-24T22:16:13,168 DEBUG [RS:0;bd6592c37bb2:40087-longCompactions-0 {}] compactions.Compactor(225): Compacting dd7740f4d1404c9a8367cc54fe2fbdd3, keycount=150, bloomtype=ROW, size=12.4 K, encoding=NONE, compression=NONE, seqNum=235, earliestPutTs=1732486570600 2024-11-24T22:16:13,171 DEBUG [RS:0;bd6592c37bb2:40087-shortCompactions-0 {}] compactions.Compactor(225): Compacting 6b1fe8f1a1474845bad917d19afeb88c, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=255, earliestPutTs=1732486570769 2024-11-24T22:16:13,171 DEBUG [RS:0;bd6592c37bb2:40087-longCompactions-0 {}] compactions.Compactor(225): Compacting 58027f2e64c3466da9d9ec87e36f74e6, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=255, earliestPutTs=1732486570769 2024-11-24T22:16:13,171 DEBUG [RS:0;bd6592c37bb2:40087-shortCompactions-0 {}] compactions.Compactor(225): Compacting 021a9989574341f2a117574e6756a07a, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=271, earliestPutTs=1732486571441 2024-11-24T22:16:13,172 DEBUG [RS:0;bd6592c37bb2:40087-longCompactions-0 {}] compactions.Compactor(225): Compacting 0ee2bcf3005f4e8197e016dfb0809d3e, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=271, earliestPutTs=1732486571441 2024-11-24T22:16:13,172 DEBUG [RS:0;bd6592c37bb2:40087-shortCompactions-0 {}] compactions.Compactor(225): Compacting 5a670e57236e459fb71bd3c4bbdd6550, keycount=200, bloomtype=ROW, size=14.4 K, encoding=NONE, compression=NONE, seqNum=282, earliestPutTs=1732486572615 2024-11-24T22:16:13,172 DEBUG [RS:0;bd6592c37bb2:40087-longCompactions-0 {}] compactions.Compactor(225): Compacting 2f3eb3c994b34515add661fedd73ef59, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=282, earliestPutTs=1732486572623 2024-11-24T22:16:13,190 INFO [RS:0;bd6592c37bb2:40087-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 744bbec6934be304e01b009be760323d#B#compaction#437 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 57.69 MB/second 2024-11-24T22:16:13,191 DEBUG [RS:0;bd6592c37bb2:40087-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/744bbec6934be304e01b009be760323d/.tmp/B/a4403f2f2fb04573be104f53fa890255 is 50, key is test_row_0/B:col10/1732486572623/Put/seqid=0 2024-11-24T22:16:13,193 INFO [RS:0;bd6592c37bb2:40087-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 744bbec6934be304e01b009be760323d#A#compaction#438 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 57.69 MB/second 2024-11-24T22:16:13,194 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45929 is added to blk_1073742341_1517 (size=13017) 2024-11-24T22:16:13,194 DEBUG [RS:0;bd6592c37bb2:40087-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/744bbec6934be304e01b009be760323d/.tmp/A/c6354d0aa5754431962b04fba90e09df is 50, key is test_row_0/A:col10/1732486572623/Put/seqid=0 2024-11-24T22:16:13,197 DEBUG [RS:0;bd6592c37bb2:40087-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/744bbec6934be304e01b009be760323d/.tmp/B/a4403f2f2fb04573be104f53fa890255 as hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/744bbec6934be304e01b009be760323d/B/a4403f2f2fb04573be104f53fa890255 2024-11-24T22:16:13,198 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45929 is added to blk_1073742342_1518 (size=13017) 2024-11-24T22:16:13,203 DEBUG [RS:0;bd6592c37bb2:40087-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/744bbec6934be304e01b009be760323d/.tmp/A/c6354d0aa5754431962b04fba90e09df as hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/744bbec6934be304e01b009be760323d/A/c6354d0aa5754431962b04fba90e09df 2024-11-24T22:16:13,205 INFO [RS:0;bd6592c37bb2:40087-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 744bbec6934be304e01b009be760323d/B of 744bbec6934be304e01b009be760323d into a4403f2f2fb04573be104f53fa890255(size=12.7 K), total size for store is 12.7 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-24T22:16:13,205 DEBUG [RS:0;bd6592c37bb2:40087-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 744bbec6934be304e01b009be760323d: 2024-11-24T22:16:13,205 INFO [RS:0;bd6592c37bb2:40087-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732486556329.744bbec6934be304e01b009be760323d., storeName=744bbec6934be304e01b009be760323d/B, priority=12, startTime=1732486573166; duration=0sec 2024-11-24T22:16:13,205 DEBUG [RS:0;bd6592c37bb2:40087-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-24T22:16:13,205 DEBUG [RS:0;bd6592c37bb2:40087-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 744bbec6934be304e01b009be760323d:B 2024-11-24T22:16:13,205 DEBUG [RS:0;bd6592c37bb2:40087-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-24T22:16:13,207 DEBUG [RS:0;bd6592c37bb2:40087-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 49484 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-24T22:16:13,207 DEBUG [RS:0;bd6592c37bb2:40087-longCompactions-0 {}] regionserver.HStore(1541): 744bbec6934be304e01b009be760323d/C is initiating minor compaction (all files) 2024-11-24T22:16:13,207 INFO [RS:0;bd6592c37bb2:40087-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 744bbec6934be304e01b009be760323d/C in TestAcidGuarantees,,1732486556329.744bbec6934be304e01b009be760323d. 2024-11-24T22:16:13,207 INFO [RS:0;bd6592c37bb2:40087-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/744bbec6934be304e01b009be760323d/C/fd0b85f893e14463833221db0d0fcbf9, hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/744bbec6934be304e01b009be760323d/C/90db8ce5d3d54446be80c89ee834828f, hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/744bbec6934be304e01b009be760323d/C/76252ddb4dd84d358abcfb118a4bb6b4, hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/744bbec6934be304e01b009be760323d/C/2583f4800d584c48b0cf73a317359c00] into tmpdir=hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/744bbec6934be304e01b009be760323d/.tmp, totalSize=48.3 K 2024-11-24T22:16:13,207 DEBUG [RS:0;bd6592c37bb2:40087-longCompactions-0 {}] compactions.Compactor(225): Compacting fd0b85f893e14463833221db0d0fcbf9, keycount=150, bloomtype=ROW, size=12.4 K, encoding=NONE, compression=NONE, seqNum=235, earliestPutTs=1732486570600 2024-11-24T22:16:13,207 INFO [RS:0;bd6592c37bb2:40087-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 744bbec6934be304e01b009be760323d/A of 744bbec6934be304e01b009be760323d into c6354d0aa5754431962b04fba90e09df(size=12.7 K), total size for store is 12.7 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-24T22:16:13,207 DEBUG [RS:0;bd6592c37bb2:40087-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 744bbec6934be304e01b009be760323d: 2024-11-24T22:16:13,207 INFO [RS:0;bd6592c37bb2:40087-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732486556329.744bbec6934be304e01b009be760323d., storeName=744bbec6934be304e01b009be760323d/A, priority=12, startTime=1732486573166; duration=0sec 2024-11-24T22:16:13,207 DEBUG [RS:0;bd6592c37bb2:40087-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-24T22:16:13,207 DEBUG [RS:0;bd6592c37bb2:40087-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 744bbec6934be304e01b009be760323d:A 2024-11-24T22:16:13,207 DEBUG [RS:0;bd6592c37bb2:40087-longCompactions-0 {}] compactions.Compactor(225): Compacting 90db8ce5d3d54446be80c89ee834828f, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=255, earliestPutTs=1732486570769 2024-11-24T22:16:13,208 DEBUG [RS:0;bd6592c37bb2:40087-longCompactions-0 {}] compactions.Compactor(225): Compacting 76252ddb4dd84d358abcfb118a4bb6b4, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=271, earliestPutTs=1732486571441 2024-11-24T22:16:13,208 DEBUG [RS:0;bd6592c37bb2:40087-longCompactions-0 {}] compactions.Compactor(225): Compacting 2583f4800d584c48b0cf73a317359c00, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=282, earliestPutTs=1732486572623 2024-11-24T22:16:13,216 INFO [RS:0;bd6592c37bb2:40087-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 744bbec6934be304e01b009be760323d#C#compaction#439 average throughput is 1.64 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 57.69 MB/second 2024-11-24T22:16:13,216 DEBUG [RS:0;bd6592c37bb2:40087-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/744bbec6934be304e01b009be760323d/.tmp/C/3516e75f79734fb39d6757fe2a86a014 is 50, key is test_row_0/C:col10/1732486572623/Put/seqid=0 2024-11-24T22:16:13,221 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45929 is added to blk_1073742343_1519 (size=13017) 2024-11-24T22:16:13,225 DEBUG [RS:0;bd6592c37bb2:40087-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/744bbec6934be304e01b009be760323d/.tmp/C/3516e75f79734fb39d6757fe2a86a014 as hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/744bbec6934be304e01b009be760323d/C/3516e75f79734fb39d6757fe2a86a014 2024-11-24T22:16:13,229 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=40087 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=134 2024-11-24T22:16:13,229 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bd6592c37bb2:0-2 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732486556329.744bbec6934be304e01b009be760323d. 2024-11-24T22:16:13,229 INFO [RS_FLUSH_OPERATIONS-regionserver/bd6592c37bb2:0-2 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.HRegion(2902): Flushing 744bbec6934be304e01b009be760323d 3/3 column families, dataSize=125.23 KB heapSize=354.02 KB 2024-11-24T22:16:13,229 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bd6592c37bb2:0-2 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 744bbec6934be304e01b009be760323d, store=A 2024-11-24T22:16:13,229 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bd6592c37bb2:0-2 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-24T22:16:13,229 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bd6592c37bb2:0-2 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 744bbec6934be304e01b009be760323d, store=B 2024-11-24T22:16:13,229 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bd6592c37bb2:0-2 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-24T22:16:13,229 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bd6592c37bb2:0-2 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 744bbec6934be304e01b009be760323d, store=C 2024-11-24T22:16:13,229 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bd6592c37bb2:0-2 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-24T22:16:13,231 INFO [RS:0;bd6592c37bb2:40087-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 744bbec6934be304e01b009be760323d/C of 744bbec6934be304e01b009be760323d into 3516e75f79734fb39d6757fe2a86a014(size=12.7 K), total size for store is 12.7 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-24T22:16:13,231 DEBUG [RS:0;bd6592c37bb2:40087-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 744bbec6934be304e01b009be760323d: 2024-11-24T22:16:13,231 INFO [RS:0;bd6592c37bb2:40087-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732486556329.744bbec6934be304e01b009be760323d., storeName=744bbec6934be304e01b009be760323d/C, priority=12, startTime=1732486573166; duration=0sec 2024-11-24T22:16:13,231 DEBUG [RS:0;bd6592c37bb2:40087-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-24T22:16:13,231 DEBUG [RS:0;bd6592c37bb2:40087-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 744bbec6934be304e01b009be760323d:C 2024-11-24T22:16:13,234 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bd6592c37bb2:0-2 {event_type=RS_FLUSH_REGIONS, pid=134}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/744bbec6934be304e01b009be760323d/.tmp/A/adba3952e07d44929aa9147ee59ac8fa is 50, key is test_row_0/A:col10/1732486572701/Put/seqid=0 2024-11-24T22:16:13,237 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45929 is added to blk_1073742344_1520 (size=12301) 2024-11-24T22:16:13,241 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37701 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=133 2024-11-24T22:16:13,359 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40087 {}] regionserver.HRegion(8855): Flush requested on 744bbec6934be304e01b009be760323d 2024-11-24T22:16:13,359 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732486556329.744bbec6934be304e01b009be760323d. as already flushing 2024-11-24T22:16:13,438 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40087 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=744bbec6934be304e01b009be760323d, server=bd6592c37bb2,40087,1732486423713 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-24T22:16:13,438 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40087 {}] ipc.CallRunner(138): callId: 67 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46820 deadline: 1732486633421, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=744bbec6934be304e01b009be760323d, server=bd6592c37bb2,40087,1732486423713 2024-11-24T22:16:13,438 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40087 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=744bbec6934be304e01b009be760323d, server=bd6592c37bb2,40087,1732486423713 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-24T22:16:13,438 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40087 {}] ipc.CallRunner(138): callId: 75 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46824 deadline: 1732486633421, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=744bbec6934be304e01b009be760323d, server=bd6592c37bb2,40087,1732486423713 2024-11-24T22:16:13,439 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732486556329.744bbec6934be304e01b009be760323d., hostname=bd6592c37bb2,40087,1732486423713, seqNum=2 , the old value is region=TestAcidGuarantees,,1732486556329.744bbec6934be304e01b009be760323d., hostname=bd6592c37bb2,40087,1732486423713, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=744bbec6934be304e01b009be760323d, server=bd6592c37bb2,40087,1732486423713 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T22:16:13,439 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40087 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=744bbec6934be304e01b009be760323d, server=bd6592c37bb2,40087,1732486423713 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-24T22:16:13,439 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732486556329.744bbec6934be304e01b009be760323d., hostname=bd6592c37bb2,40087,1732486423713, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=744bbec6934be304e01b009be760323d, server=bd6592c37bb2,40087,1732486423713 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T22:16:13,439 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732486556329.744bbec6934be304e01b009be760323d., hostname=bd6592c37bb2,40087,1732486423713, seqNum=2 because the exception is null or not the one we care about 2024-11-24T22:16:13,439 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40087 {}] ipc.CallRunner(138): callId: 66 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46816 deadline: 1732486633421, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=744bbec6934be304e01b009be760323d, server=bd6592c37bb2,40087,1732486423713 2024-11-24T22:16:13,439 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732486556329.744bbec6934be304e01b009be760323d., hostname=bd6592c37bb2,40087,1732486423713, seqNum=2 , the old value is region=TestAcidGuarantees,,1732486556329.744bbec6934be304e01b009be760323d., hostname=bd6592c37bb2,40087,1732486423713, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=744bbec6934be304e01b009be760323d, server=bd6592c37bb2,40087,1732486423713 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T22:16:13,439 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732486556329.744bbec6934be304e01b009be760323d., hostname=bd6592c37bb2,40087,1732486423713, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=744bbec6934be304e01b009be760323d, server=bd6592c37bb2,40087,1732486423713 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T22:16:13,439 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732486556329.744bbec6934be304e01b009be760323d., hostname=bd6592c37bb2,40087,1732486423713, seqNum=2 because the exception is null or not the one we care about 2024-11-24T22:16:13,439 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40087 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=744bbec6934be304e01b009be760323d, server=bd6592c37bb2,40087,1732486423713 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-24T22:16:13,439 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40087 {}] ipc.CallRunner(138): callId: 147 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46792 deadline: 1732486633436, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=744bbec6934be304e01b009be760323d, server=bd6592c37bb2,40087,1732486423713 2024-11-24T22:16:13,439 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732486556329.744bbec6934be304e01b009be760323d., hostname=bd6592c37bb2,40087,1732486423713, seqNum=2 , the old value is region=TestAcidGuarantees,,1732486556329.744bbec6934be304e01b009be760323d., hostname=bd6592c37bb2,40087,1732486423713, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=744bbec6934be304e01b009be760323d, server=bd6592c37bb2,40087,1732486423713 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T22:16:13,439 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732486556329.744bbec6934be304e01b009be760323d., hostname=bd6592c37bb2,40087,1732486423713, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=744bbec6934be304e01b009be760323d, server=bd6592c37bb2,40087,1732486423713 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T22:16:13,439 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732486556329.744bbec6934be304e01b009be760323d., hostname=bd6592c37bb2,40087,1732486423713, seqNum=2 because the exception is null or not the one we care about 2024-11-24T22:16:13,440 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732486556329.744bbec6934be304e01b009be760323d., hostname=bd6592c37bb2,40087,1732486423713, seqNum=2 , the old value is region=TestAcidGuarantees,,1732486556329.744bbec6934be304e01b009be760323d., hostname=bd6592c37bb2,40087,1732486423713, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=744bbec6934be304e01b009be760323d, server=bd6592c37bb2,40087,1732486423713 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T22:16:13,440 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732486556329.744bbec6934be304e01b009be760323d., hostname=bd6592c37bb2,40087,1732486423713, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=744bbec6934be304e01b009be760323d, server=bd6592c37bb2,40087,1732486423713 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T22:16:13,440 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732486556329.744bbec6934be304e01b009be760323d., hostname=bd6592c37bb2,40087,1732486423713, seqNum=2 because the exception is null or not the one we care about 2024-11-24T22:16:13,455 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40087 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=744bbec6934be304e01b009be760323d, server=bd6592c37bb2,40087,1732486423713 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-24T22:16:13,455 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40087 {}] ipc.CallRunner(138): callId: 72 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46842 deadline: 1732486633438, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=744bbec6934be304e01b009be760323d, server=bd6592c37bb2,40087,1732486423713 2024-11-24T22:16:13,455 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732486556329.744bbec6934be304e01b009be760323d., hostname=bd6592c37bb2,40087,1732486423713, seqNum=2 , the old value is region=TestAcidGuarantees,,1732486556329.744bbec6934be304e01b009be760323d., hostname=bd6592c37bb2,40087,1732486423713, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=744bbec6934be304e01b009be760323d, server=bd6592c37bb2,40087,1732486423713 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T22:16:13,455 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732486556329.744bbec6934be304e01b009be760323d., hostname=bd6592c37bb2,40087,1732486423713, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=744bbec6934be304e01b009be760323d, server=bd6592c37bb2,40087,1732486423713 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T22:16:13,455 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732486556329.744bbec6934be304e01b009be760323d., hostname=bd6592c37bb2,40087,1732486423713, seqNum=2 because the exception is null or not the one we care about 2024-11-24T22:16:13,557 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40087 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=744bbec6934be304e01b009be760323d, server=bd6592c37bb2,40087,1732486423713 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-24T22:16:13,558 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40087 {}] ipc.CallRunner(138): callId: 148 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46792 deadline: 1732486633551, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=744bbec6934be304e01b009be760323d, server=bd6592c37bb2,40087,1732486423713 2024-11-24T22:16:13,558 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732486556329.744bbec6934be304e01b009be760323d., hostname=bd6592c37bb2,40087,1732486423713, seqNum=2 , the old value is region=TestAcidGuarantees,,1732486556329.744bbec6934be304e01b009be760323d., hostname=bd6592c37bb2,40087,1732486423713, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=744bbec6934be304e01b009be760323d, server=bd6592c37bb2,40087,1732486423713 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T22:16:13,558 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732486556329.744bbec6934be304e01b009be760323d., hostname=bd6592c37bb2,40087,1732486423713, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=744bbec6934be304e01b009be760323d, server=bd6592c37bb2,40087,1732486423713 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T22:16:13,558 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732486556329.744bbec6934be304e01b009be760323d., hostname=bd6592c37bb2,40087,1732486423713, seqNum=2 because the exception is null or not the one we care about 2024-11-24T22:16:13,575 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40087 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=744bbec6934be304e01b009be760323d, server=bd6592c37bb2,40087,1732486423713 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-24T22:16:13,575 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40087 {}] ipc.CallRunner(138): callId: 73 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46842 deadline: 1732486633560, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=744bbec6934be304e01b009be760323d, server=bd6592c37bb2,40087,1732486423713 2024-11-24T22:16:13,575 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732486556329.744bbec6934be304e01b009be760323d., hostname=bd6592c37bb2,40087,1732486423713, seqNum=2 , the old value is region=TestAcidGuarantees,,1732486556329.744bbec6934be304e01b009be760323d., hostname=bd6592c37bb2,40087,1732486423713, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=744bbec6934be304e01b009be760323d, server=bd6592c37bb2,40087,1732486423713 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T22:16:13,575 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732486556329.744bbec6934be304e01b009be760323d., hostname=bd6592c37bb2,40087,1732486423713, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=744bbec6934be304e01b009be760323d, server=bd6592c37bb2,40087,1732486423713 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T22:16:13,575 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732486556329.744bbec6934be304e01b009be760323d., hostname=bd6592c37bb2,40087,1732486423713, seqNum=2 because the exception is null or not the one we care about 2024-11-24T22:16:13,638 INFO [RS_FLUSH_OPERATIONS-regionserver/bd6592c37bb2:0-2 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=33.54 KB at sequenceid=313 (bloomFilter=true), to=hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/744bbec6934be304e01b009be760323d/.tmp/A/adba3952e07d44929aa9147ee59ac8fa 2024-11-24T22:16:13,646 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bd6592c37bb2:0-2 {event_type=RS_FLUSH_REGIONS, pid=134}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/744bbec6934be304e01b009be760323d/.tmp/B/7e1b795cbaa54f958590c9eee2e5bbf9 is 50, key is test_row_0/B:col10/1732486572701/Put/seqid=0 2024-11-24T22:16:13,650 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45929 is added to blk_1073742345_1521 (size=12301) 2024-11-24T22:16:13,751 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37701 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=133 2024-11-24T22:16:13,775 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40087 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=744bbec6934be304e01b009be760323d, server=bd6592c37bb2,40087,1732486423713 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-24T22:16:13,775 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40087 {}] ipc.CallRunner(138): callId: 149 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46792 deadline: 1732486633763, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=744bbec6934be304e01b009be760323d, server=bd6592c37bb2,40087,1732486423713 2024-11-24T22:16:13,776 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732486556329.744bbec6934be304e01b009be760323d., hostname=bd6592c37bb2,40087,1732486423713, seqNum=2 , the old value is region=TestAcidGuarantees,,1732486556329.744bbec6934be304e01b009be760323d., hostname=bd6592c37bb2,40087,1732486423713, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=744bbec6934be304e01b009be760323d, server=bd6592c37bb2,40087,1732486423713 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T22:16:13,776 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732486556329.744bbec6934be304e01b009be760323d., hostname=bd6592c37bb2,40087,1732486423713, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=744bbec6934be304e01b009be760323d, server=bd6592c37bb2,40087,1732486423713 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T22:16:13,776 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732486556329.744bbec6934be304e01b009be760323d., hostname=bd6592c37bb2,40087,1732486423713, seqNum=2 because the exception is null or not the one we care about 2024-11-24T22:16:13,790 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40087 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=744bbec6934be304e01b009be760323d, server=bd6592c37bb2,40087,1732486423713 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-24T22:16:13,790 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40087 {}] ipc.CallRunner(138): callId: 74 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46842 deadline: 1732486633781, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=744bbec6934be304e01b009be760323d, server=bd6592c37bb2,40087,1732486423713 2024-11-24T22:16:13,791 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732486556329.744bbec6934be304e01b009be760323d., hostname=bd6592c37bb2,40087,1732486423713, seqNum=2 , the old value is region=TestAcidGuarantees,,1732486556329.744bbec6934be304e01b009be760323d., hostname=bd6592c37bb2,40087,1732486423713, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=744bbec6934be304e01b009be760323d, server=bd6592c37bb2,40087,1732486423713 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T22:16:13,791 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732486556329.744bbec6934be304e01b009be760323d., hostname=bd6592c37bb2,40087,1732486423713, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=744bbec6934be304e01b009be760323d, server=bd6592c37bb2,40087,1732486423713 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T22:16:13,791 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732486556329.744bbec6934be304e01b009be760323d., hostname=bd6592c37bb2,40087,1732486423713, seqNum=2 because the exception is null or not the one we care about 2024-11-24T22:16:13,957 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40087 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=744bbec6934be304e01b009be760323d, server=bd6592c37bb2,40087,1732486423713 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-24T22:16:13,958 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40087 {}] ipc.CallRunner(138): callId: 68 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46820 deadline: 1732486633951, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=744bbec6934be304e01b009be760323d, server=bd6592c37bb2,40087,1732486423713 2024-11-24T22:16:13,958 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40087 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=744bbec6934be304e01b009be760323d, server=bd6592c37bb2,40087,1732486423713 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-24T22:16:13,958 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732486556329.744bbec6934be304e01b009be760323d., hostname=bd6592c37bb2,40087,1732486423713, seqNum=2 , the old value is region=TestAcidGuarantees,,1732486556329.744bbec6934be304e01b009be760323d., hostname=bd6592c37bb2,40087,1732486423713, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=744bbec6934be304e01b009be760323d, server=bd6592c37bb2,40087,1732486423713 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T22:16:13,958 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40087 {}] ipc.CallRunner(138): callId: 76 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46824 deadline: 1732486633951, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=744bbec6934be304e01b009be760323d, server=bd6592c37bb2,40087,1732486423713 2024-11-24T22:16:13,958 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732486556329.744bbec6934be304e01b009be760323d., hostname=bd6592c37bb2,40087,1732486423713, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=744bbec6934be304e01b009be760323d, server=bd6592c37bb2,40087,1732486423713 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T22:16:13,958 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732486556329.744bbec6934be304e01b009be760323d., hostname=bd6592c37bb2,40087,1732486423713, seqNum=2 because the exception is null or not the one we care about 2024-11-24T22:16:13,958 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40087 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=744bbec6934be304e01b009be760323d, server=bd6592c37bb2,40087,1732486423713 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-24T22:16:13,958 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732486556329.744bbec6934be304e01b009be760323d., hostname=bd6592c37bb2,40087,1732486423713, seqNum=2 , the old value is region=TestAcidGuarantees,,1732486556329.744bbec6934be304e01b009be760323d., hostname=bd6592c37bb2,40087,1732486423713, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=744bbec6934be304e01b009be760323d, server=bd6592c37bb2,40087,1732486423713 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T22:16:13,958 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732486556329.744bbec6934be304e01b009be760323d., hostname=bd6592c37bb2,40087,1732486423713, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=744bbec6934be304e01b009be760323d, server=bd6592c37bb2,40087,1732486423713 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T22:16:13,958 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40087 {}] ipc.CallRunner(138): callId: 67 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46816 deadline: 1732486633951, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=744bbec6934be304e01b009be760323d, server=bd6592c37bb2,40087,1732486423713 2024-11-24T22:16:13,958 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732486556329.744bbec6934be304e01b009be760323d., hostname=bd6592c37bb2,40087,1732486423713, seqNum=2 because the exception is null or not the one we care about 2024-11-24T22:16:13,959 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732486556329.744bbec6934be304e01b009be760323d., hostname=bd6592c37bb2,40087,1732486423713, seqNum=2 , the old value is region=TestAcidGuarantees,,1732486556329.744bbec6934be304e01b009be760323d., hostname=bd6592c37bb2,40087,1732486423713, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=744bbec6934be304e01b009be760323d, server=bd6592c37bb2,40087,1732486423713 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T22:16:13,959 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732486556329.744bbec6934be304e01b009be760323d., hostname=bd6592c37bb2,40087,1732486423713, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=744bbec6934be304e01b009be760323d, server=bd6592c37bb2,40087,1732486423713 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T22:16:13,959 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732486556329.744bbec6934be304e01b009be760323d., hostname=bd6592c37bb2,40087,1732486423713, seqNum=2 because the exception is null or not the one we care about 2024-11-24T22:16:14,051 INFO [RS_FLUSH_OPERATIONS-regionserver/bd6592c37bb2:0-2 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=33.54 KB at sequenceid=313 (bloomFilter=true), to=hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/744bbec6934be304e01b009be760323d/.tmp/B/7e1b795cbaa54f958590c9eee2e5bbf9 2024-11-24T22:16:14,058 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bd6592c37bb2:0-2 {event_type=RS_FLUSH_REGIONS, pid=134}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/744bbec6934be304e01b009be760323d/.tmp/C/3e9d21519ab34af298726d4eedaf4608 is 50, key is test_row_0/C:col10/1732486572701/Put/seqid=0 2024-11-24T22:16:14,062 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45929 is added to blk_1073742346_1522 (size=12301) 2024-11-24T22:16:14,091 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40087 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=744bbec6934be304e01b009be760323d, server=bd6592c37bb2,40087,1732486423713 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-24T22:16:14,091 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40087 {}] ipc.CallRunner(138): callId: 150 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46792 deadline: 1732486634080, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=744bbec6934be304e01b009be760323d, server=bd6592c37bb2,40087,1732486423713 2024-11-24T22:16:14,091 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732486556329.744bbec6934be304e01b009be760323d., hostname=bd6592c37bb2,40087,1732486423713, seqNum=2 , the old value is region=TestAcidGuarantees,,1732486556329.744bbec6934be304e01b009be760323d., hostname=bd6592c37bb2,40087,1732486423713, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=744bbec6934be304e01b009be760323d, server=bd6592c37bb2,40087,1732486423713 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T22:16:14,091 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732486556329.744bbec6934be304e01b009be760323d., hostname=bd6592c37bb2,40087,1732486423713, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=744bbec6934be304e01b009be760323d, server=bd6592c37bb2,40087,1732486423713 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T22:16:14,091 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732486556329.744bbec6934be304e01b009be760323d., hostname=bd6592c37bb2,40087,1732486423713, seqNum=2 because the exception is null or not the one we care about 2024-11-24T22:16:14,106 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40087 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=744bbec6934be304e01b009be760323d, server=bd6592c37bb2,40087,1732486423713 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-24T22:16:14,106 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40087 {}] ipc.CallRunner(138): callId: 75 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46842 deadline: 1732486634101, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=744bbec6934be304e01b009be760323d, server=bd6592c37bb2,40087,1732486423713 2024-11-24T22:16:14,107 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732486556329.744bbec6934be304e01b009be760323d., hostname=bd6592c37bb2,40087,1732486423713, seqNum=2 , the old value is region=TestAcidGuarantees,,1732486556329.744bbec6934be304e01b009be760323d., hostname=bd6592c37bb2,40087,1732486423713, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=744bbec6934be304e01b009be760323d, server=bd6592c37bb2,40087,1732486423713 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T22:16:14,107 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732486556329.744bbec6934be304e01b009be760323d., hostname=bd6592c37bb2,40087,1732486423713, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=744bbec6934be304e01b009be760323d, server=bd6592c37bb2,40087,1732486423713 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T22:16:14,107 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732486556329.744bbec6934be304e01b009be760323d., hostname=bd6592c37bb2,40087,1732486423713, seqNum=2 because the exception is null or not the one we care about 2024-11-24T22:16:14,463 INFO [RS_FLUSH_OPERATIONS-regionserver/bd6592c37bb2:0-2 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=58.14 KB at sequenceid=313 (bloomFilter=true), to=hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/744bbec6934be304e01b009be760323d/.tmp/C/3e9d21519ab34af298726d4eedaf4608 2024-11-24T22:16:14,468 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bd6592c37bb2:0-2 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/744bbec6934be304e01b009be760323d/.tmp/A/adba3952e07d44929aa9147ee59ac8fa as hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/744bbec6934be304e01b009be760323d/A/adba3952e07d44929aa9147ee59ac8fa 2024-11-24T22:16:14,472 INFO [RS_FLUSH_OPERATIONS-regionserver/bd6592c37bb2:0-2 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/744bbec6934be304e01b009be760323d/A/adba3952e07d44929aa9147ee59ac8fa, entries=150, sequenceid=313, filesize=12.0 K 2024-11-24T22:16:14,473 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bd6592c37bb2:0-2 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/744bbec6934be304e01b009be760323d/.tmp/B/7e1b795cbaa54f958590c9eee2e5bbf9 as hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/744bbec6934be304e01b009be760323d/B/7e1b795cbaa54f958590c9eee2e5bbf9 2024-11-24T22:16:14,477 INFO [RS_FLUSH_OPERATIONS-regionserver/bd6592c37bb2:0-2 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/744bbec6934be304e01b009be760323d/B/7e1b795cbaa54f958590c9eee2e5bbf9, entries=150, sequenceid=313, filesize=12.0 K 2024-11-24T22:16:14,478 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bd6592c37bb2:0-2 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/744bbec6934be304e01b009be760323d/.tmp/C/3e9d21519ab34af298726d4eedaf4608 as hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/744bbec6934be304e01b009be760323d/C/3e9d21519ab34af298726d4eedaf4608 2024-11-24T22:16:14,488 INFO [RS_FLUSH_OPERATIONS-regionserver/bd6592c37bb2:0-2 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/744bbec6934be304e01b009be760323d/C/3e9d21519ab34af298726d4eedaf4608, entries=150, sequenceid=313, filesize=12.0 K 2024-11-24T22:16:14,489 INFO [RS_FLUSH_OPERATIONS-regionserver/bd6592c37bb2:0-2 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.HRegion(3140): Finished flush of dataSize ~125.23 KB/128240, heapSize ~353.97 KB/362464, currentSize=60.38 KB/61830 for 744bbec6934be304e01b009be760323d in 1260ms, sequenceid=313, compaction requested=false 2024-11-24T22:16:14,489 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bd6592c37bb2:0-2 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.HRegion(2603): Flush status journal for 744bbec6934be304e01b009be760323d: 2024-11-24T22:16:14,489 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bd6592c37bb2:0-2 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732486556329.744bbec6934be304e01b009be760323d. 2024-11-24T22:16:14,489 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bd6592c37bb2:0-2 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=134 2024-11-24T22:16:14,492 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37701 {}] master.HMaster(4169): Remote procedure done, pid=134 2024-11-24T22:16:14,502 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=134, resume processing ppid=133 2024-11-24T22:16:14,502 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=134, ppid=133, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.8840 sec 2024-11-24T22:16:14,504 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=133, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=133, table=TestAcidGuarantees in 1.8910 sec 2024-11-24T22:16:14,616 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40087 {}] regionserver.HRegion(8855): Flush requested on 744bbec6934be304e01b009be760323d 2024-11-24T22:16:14,616 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 744bbec6934be304e01b009be760323d 3/3 column families, dataSize=67.09 KB heapSize=199.97 KB 2024-11-24T22:16:14,616 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 744bbec6934be304e01b009be760323d, store=A 2024-11-24T22:16:14,616 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-24T22:16:14,616 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 744bbec6934be304e01b009be760323d, store=B 2024-11-24T22:16:14,616 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-24T22:16:14,616 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 744bbec6934be304e01b009be760323d, store=C 2024-11-24T22:16:14,617 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-24T22:16:14,623 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/744bbec6934be304e01b009be760323d/.tmp/A/3bcf468781fb40189fb54b126de0bb40 is 50, key is test_row_0/A:col10/1732486573413/Put/seqid=0 2024-11-24T22:16:14,631 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45929 is added to blk_1073742347_1523 (size=14741) 2024-11-24T22:16:14,632 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=327 (bloomFilter=true), to=hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/744bbec6934be304e01b009be760323d/.tmp/A/3bcf468781fb40189fb54b126de0bb40 2024-11-24T22:16:14,641 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/744bbec6934be304e01b009be760323d/.tmp/B/3af7635b011644278e969391287c6f08 is 50, key is test_row_0/B:col10/1732486573413/Put/seqid=0 2024-11-24T22:16:14,645 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45929 is added to blk_1073742348_1524 (size=12301) 2024-11-24T22:16:14,645 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=327 (bloomFilter=true), to=hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/744bbec6934be304e01b009be760323d/.tmp/B/3af7635b011644278e969391287c6f08 2024-11-24T22:16:14,662 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/744bbec6934be304e01b009be760323d/.tmp/C/5325c2d24c764c4eb297525675256fbd is 50, key is test_row_0/C:col10/1732486573413/Put/seqid=0 2024-11-24T22:16:14,670 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45929 is added to blk_1073742349_1525 (size=12301) 2024-11-24T22:16:14,671 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=327 (bloomFilter=true), to=hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/744bbec6934be304e01b009be760323d/.tmp/C/5325c2d24c764c4eb297525675256fbd 2024-11-24T22:16:14,690 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/744bbec6934be304e01b009be760323d/.tmp/A/3bcf468781fb40189fb54b126de0bb40 as hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/744bbec6934be304e01b009be760323d/A/3bcf468781fb40189fb54b126de0bb40 2024-11-24T22:16:14,694 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/744bbec6934be304e01b009be760323d/A/3bcf468781fb40189fb54b126de0bb40, entries=200, sequenceid=327, filesize=14.4 K 2024-11-24T22:16:14,694 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/744bbec6934be304e01b009be760323d/.tmp/B/3af7635b011644278e969391287c6f08 as hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/744bbec6934be304e01b009be760323d/B/3af7635b011644278e969391287c6f08 2024-11-24T22:16:14,697 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/744bbec6934be304e01b009be760323d/B/3af7635b011644278e969391287c6f08, entries=150, sequenceid=327, filesize=12.0 K 2024-11-24T22:16:14,698 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/744bbec6934be304e01b009be760323d/.tmp/C/5325c2d24c764c4eb297525675256fbd as hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/744bbec6934be304e01b009be760323d/C/5325c2d24c764c4eb297525675256fbd 2024-11-24T22:16:14,704 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/744bbec6934be304e01b009be760323d/C/5325c2d24c764c4eb297525675256fbd, entries=150, sequenceid=327, filesize=12.0 K 2024-11-24T22:16:14,706 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~67.09 KB/68700, heapSize ~199.92 KB/204720, currentSize=87.22 KB/89310 for 744bbec6934be304e01b009be760323d in 90ms, sequenceid=327, compaction requested=true 2024-11-24T22:16:14,706 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 744bbec6934be304e01b009be760323d: 2024-11-24T22:16:14,707 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 744bbec6934be304e01b009be760323d:A, priority=-2147483648, current under compaction store size is 1 2024-11-24T22:16:14,707 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-24T22:16:14,707 DEBUG [RS:0;bd6592c37bb2:40087-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-24T22:16:14,707 DEBUG [RS:0;bd6592c37bb2:40087-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-24T22:16:14,707 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 744bbec6934be304e01b009be760323d:B, priority=-2147483648, current under compaction store size is 2 2024-11-24T22:16:14,707 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-24T22:16:14,707 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 744bbec6934be304e01b009be760323d:C, priority=-2147483648, current under compaction store size is 3 2024-11-24T22:16:14,707 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-24T22:16:14,709 DEBUG [RS:0;bd6592c37bb2:40087-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37619 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-24T22:16:14,709 DEBUG [RS:0;bd6592c37bb2:40087-longCompactions-0 {}] regionserver.HStore(1541): 744bbec6934be304e01b009be760323d/B is initiating minor compaction (all files) 2024-11-24T22:16:14,709 INFO [RS:0;bd6592c37bb2:40087-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 744bbec6934be304e01b009be760323d/B in TestAcidGuarantees,,1732486556329.744bbec6934be304e01b009be760323d. 2024-11-24T22:16:14,709 INFO [RS:0;bd6592c37bb2:40087-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/744bbec6934be304e01b009be760323d/B/a4403f2f2fb04573be104f53fa890255, hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/744bbec6934be304e01b009be760323d/B/7e1b795cbaa54f958590c9eee2e5bbf9, hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/744bbec6934be304e01b009be760323d/B/3af7635b011644278e969391287c6f08] into tmpdir=hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/744bbec6934be304e01b009be760323d/.tmp, totalSize=36.7 K 2024-11-24T22:16:14,709 DEBUG [RS:0;bd6592c37bb2:40087-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 40059 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-24T22:16:14,709 DEBUG [RS:0;bd6592c37bb2:40087-shortCompactions-0 {}] regionserver.HStore(1541): 744bbec6934be304e01b009be760323d/A is initiating minor compaction (all files) 2024-11-24T22:16:14,709 INFO [RS:0;bd6592c37bb2:40087-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 744bbec6934be304e01b009be760323d/A in TestAcidGuarantees,,1732486556329.744bbec6934be304e01b009be760323d. 2024-11-24T22:16:14,709 INFO [RS:0;bd6592c37bb2:40087-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/744bbec6934be304e01b009be760323d/A/c6354d0aa5754431962b04fba90e09df, hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/744bbec6934be304e01b009be760323d/A/adba3952e07d44929aa9147ee59ac8fa, hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/744bbec6934be304e01b009be760323d/A/3bcf468781fb40189fb54b126de0bb40] into tmpdir=hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/744bbec6934be304e01b009be760323d/.tmp, totalSize=39.1 K 2024-11-24T22:16:14,710 DEBUG [RS:0;bd6592c37bb2:40087-shortCompactions-0 {}] compactions.Compactor(225): Compacting c6354d0aa5754431962b04fba90e09df, keycount=150, bloomtype=ROW, size=12.7 K, encoding=NONE, compression=NONE, seqNum=282, earliestPutTs=1732486572623 2024-11-24T22:16:14,710 DEBUG [RS:0;bd6592c37bb2:40087-longCompactions-0 {}] compactions.Compactor(225): Compacting a4403f2f2fb04573be104f53fa890255, keycount=150, bloomtype=ROW, size=12.7 K, encoding=NONE, compression=NONE, seqNum=282, earliestPutTs=1732486572623 2024-11-24T22:16:14,711 DEBUG [RS:0;bd6592c37bb2:40087-longCompactions-0 {}] compactions.Compactor(225): Compacting 7e1b795cbaa54f958590c9eee2e5bbf9, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=313, earliestPutTs=1732486572701 2024-11-24T22:16:14,712 DEBUG [RS:0;bd6592c37bb2:40087-shortCompactions-0 {}] compactions.Compactor(225): Compacting adba3952e07d44929aa9147ee59ac8fa, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=313, earliestPutTs=1732486572701 2024-11-24T22:16:14,712 DEBUG [RS:0;bd6592c37bb2:40087-longCompactions-0 {}] compactions.Compactor(225): Compacting 3af7635b011644278e969391287c6f08, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=327, earliestPutTs=1732486573413 2024-11-24T22:16:14,712 DEBUG [RS:0;bd6592c37bb2:40087-shortCompactions-0 {}] compactions.Compactor(225): Compacting 3bcf468781fb40189fb54b126de0bb40, keycount=200, bloomtype=ROW, size=14.4 K, encoding=NONE, compression=NONE, seqNum=327, earliestPutTs=1732486573413 2024-11-24T22:16:14,712 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 744bbec6934be304e01b009be760323d 3/3 column families, dataSize=93.93 KB heapSize=279.66 KB 2024-11-24T22:16:14,712 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 744bbec6934be304e01b009be760323d, store=A 2024-11-24T22:16:14,712 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-24T22:16:14,712 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 744bbec6934be304e01b009be760323d, store=B 2024-11-24T22:16:14,713 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-24T22:16:14,713 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 744bbec6934be304e01b009be760323d, store=C 2024-11-24T22:16:14,713 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-24T22:16:14,713 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40087 {}] regionserver.HRegion(8855): Flush requested on 744bbec6934be304e01b009be760323d 2024-11-24T22:16:14,720 INFO [RS:0;bd6592c37bb2:40087-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 744bbec6934be304e01b009be760323d#A#compaction#446 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 57.69 MB/second 2024-11-24T22:16:14,721 DEBUG [RS:0;bd6592c37bb2:40087-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/744bbec6934be304e01b009be760323d/.tmp/A/4cbb2b50c7d74794986c166190e79fdb is 50, key is test_row_0/A:col10/1732486573413/Put/seqid=0 2024-11-24T22:16:14,727 INFO [RS:0;bd6592c37bb2:40087-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 744bbec6934be304e01b009be760323d#B#compaction#447 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 57.69 MB/second 2024-11-24T22:16:14,727 DEBUG [RS:0;bd6592c37bb2:40087-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/744bbec6934be304e01b009be760323d/.tmp/B/6ee3d0de54754f999dfa2ddfe20f7a94 is 50, key is test_row_0/B:col10/1732486573413/Put/seqid=0 2024-11-24T22:16:14,735 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/744bbec6934be304e01b009be760323d/.tmp/A/8007d32c8dc5404ea79a5762cb35d83d is 50, key is test_row_0/A:col10/1732486574698/Put/seqid=0 2024-11-24T22:16:14,747 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45929 is added to blk_1073742350_1526 (size=13119) 2024-11-24T22:16:14,748 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45929 is added to blk_1073742352_1528 (size=12301) 2024-11-24T22:16:14,748 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45929 is added to blk_1073742351_1527 (size=13119) 2024-11-24T22:16:14,753 DEBUG [RS:0;bd6592c37bb2:40087-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/744bbec6934be304e01b009be760323d/.tmp/B/6ee3d0de54754f999dfa2ddfe20f7a94 as hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/744bbec6934be304e01b009be760323d/B/6ee3d0de54754f999dfa2ddfe20f7a94 2024-11-24T22:16:14,758 INFO [RS:0;bd6592c37bb2:40087-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 744bbec6934be304e01b009be760323d/B of 744bbec6934be304e01b009be760323d into 6ee3d0de54754f999dfa2ddfe20f7a94(size=12.8 K), total size for store is 12.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-24T22:16:14,758 DEBUG [RS:0;bd6592c37bb2:40087-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 744bbec6934be304e01b009be760323d: 2024-11-24T22:16:14,758 INFO [RS:0;bd6592c37bb2:40087-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732486556329.744bbec6934be304e01b009be760323d., storeName=744bbec6934be304e01b009be760323d/B, priority=13, startTime=1732486574707; duration=0sec 2024-11-24T22:16:14,758 DEBUG [RS:0;bd6592c37bb2:40087-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-24T22:16:14,758 DEBUG [RS:0;bd6592c37bb2:40087-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 744bbec6934be304e01b009be760323d:B 2024-11-24T22:16:14,758 DEBUG [RS:0;bd6592c37bb2:40087-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-24T22:16:14,759 DEBUG [RS:0;bd6592c37bb2:40087-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37619 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-24T22:16:14,759 DEBUG [RS:0;bd6592c37bb2:40087-longCompactions-0 {}] regionserver.HStore(1541): 744bbec6934be304e01b009be760323d/C is initiating minor compaction (all files) 2024-11-24T22:16:14,759 INFO [RS:0;bd6592c37bb2:40087-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 744bbec6934be304e01b009be760323d/C in TestAcidGuarantees,,1732486556329.744bbec6934be304e01b009be760323d. 2024-11-24T22:16:14,759 INFO [RS:0;bd6592c37bb2:40087-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/744bbec6934be304e01b009be760323d/C/3516e75f79734fb39d6757fe2a86a014, hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/744bbec6934be304e01b009be760323d/C/3e9d21519ab34af298726d4eedaf4608, hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/744bbec6934be304e01b009be760323d/C/5325c2d24c764c4eb297525675256fbd] into tmpdir=hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/744bbec6934be304e01b009be760323d/.tmp, totalSize=36.7 K 2024-11-24T22:16:14,760 DEBUG [RS:0;bd6592c37bb2:40087-longCompactions-0 {}] compactions.Compactor(225): Compacting 3516e75f79734fb39d6757fe2a86a014, keycount=150, bloomtype=ROW, size=12.7 K, encoding=NONE, compression=NONE, seqNum=282, earliestPutTs=1732486572623 2024-11-24T22:16:14,760 DEBUG [RS:0;bd6592c37bb2:40087-longCompactions-0 {}] compactions.Compactor(225): Compacting 3e9d21519ab34af298726d4eedaf4608, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=313, earliestPutTs=1732486572701 2024-11-24T22:16:14,760 DEBUG [RS:0;bd6592c37bb2:40087-longCompactions-0 {}] compactions.Compactor(225): Compacting 5325c2d24c764c4eb297525675256fbd, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=327, earliestPutTs=1732486573413 2024-11-24T22:16:14,763 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37701 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=133 2024-11-24T22:16:14,764 INFO [RPCClient-NioEventLoopGroup-4-10 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-24T22:16:14,764 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37701 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-24T22:16:14,765 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37701 {}] procedure2.ProcedureExecutor(1139): Stored pid=135, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=135, table=TestAcidGuarantees 2024-11-24T22:16:14,766 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37701 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=135 2024-11-24T22:16:14,766 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=135, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=135, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-24T22:16:14,767 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=135, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=135, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-24T22:16:14,767 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=136, ppid=135, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-24T22:16:14,778 INFO [RS:0;bd6592c37bb2:40087-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 744bbec6934be304e01b009be760323d#C#compaction#449 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 57.69 MB/second 2024-11-24T22:16:14,779 DEBUG [RS:0;bd6592c37bb2:40087-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/744bbec6934be304e01b009be760323d/.tmp/C/18cfa9a432cd4a559609140d0121d730 is 50, key is test_row_0/C:col10/1732486573413/Put/seqid=0 2024-11-24T22:16:14,786 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45929 is added to blk_1073742353_1529 (size=13119) 2024-11-24T22:16:14,807 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40087 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=744bbec6934be304e01b009be760323d, server=bd6592c37bb2,40087,1732486423713 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-24T22:16:14,807 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40087 {}] ipc.CallRunner(138): callId: 89 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46842 deadline: 1732486634794, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=744bbec6934be304e01b009be760323d, server=bd6592c37bb2,40087,1732486423713 2024-11-24T22:16:14,807 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732486556329.744bbec6934be304e01b009be760323d., hostname=bd6592c37bb2,40087,1732486423713, seqNum=2 , the old value is region=TestAcidGuarantees,,1732486556329.744bbec6934be304e01b009be760323d., hostname=bd6592c37bb2,40087,1732486423713, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=744bbec6934be304e01b009be760323d, server=bd6592c37bb2,40087,1732486423713 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T22:16:14,807 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732486556329.744bbec6934be304e01b009be760323d., hostname=bd6592c37bb2,40087,1732486423713, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=744bbec6934be304e01b009be760323d, server=bd6592c37bb2,40087,1732486423713 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T22:16:14,807 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732486556329.744bbec6934be304e01b009be760323d., hostname=bd6592c37bb2,40087,1732486423713, seqNum=2 because the exception is null or not the one we care about 2024-11-24T22:16:14,813 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40087 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=744bbec6934be304e01b009be760323d, server=bd6592c37bb2,40087,1732486423713 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-24T22:16:14,813 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40087 {}] ipc.CallRunner(138): callId: 165 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46792 deadline: 1732486634806, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=744bbec6934be304e01b009be760323d, server=bd6592c37bb2,40087,1732486423713 2024-11-24T22:16:14,814 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732486556329.744bbec6934be304e01b009be760323d., hostname=bd6592c37bb2,40087,1732486423713, seqNum=2 , the old value is region=TestAcidGuarantees,,1732486556329.744bbec6934be304e01b009be760323d., hostname=bd6592c37bb2,40087,1732486423713, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=744bbec6934be304e01b009be760323d, server=bd6592c37bb2,40087,1732486423713 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T22:16:14,814 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732486556329.744bbec6934be304e01b009be760323d., hostname=bd6592c37bb2,40087,1732486423713, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=744bbec6934be304e01b009be760323d, server=bd6592c37bb2,40087,1732486423713 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T22:16:14,814 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732486556329.744bbec6934be304e01b009be760323d., hostname=bd6592c37bb2,40087,1732486423713, seqNum=2 because the exception is null or not the one we care about 2024-11-24T22:16:14,871 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37701 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=135 2024-11-24T22:16:14,916 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40087 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=744bbec6934be304e01b009be760323d, server=bd6592c37bb2,40087,1732486423713 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-24T22:16:14,916 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40087 {}] ipc.CallRunner(138): callId: 90 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46842 deadline: 1732486634911, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=744bbec6934be304e01b009be760323d, server=bd6592c37bb2,40087,1732486423713 2024-11-24T22:16:14,916 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732486556329.744bbec6934be304e01b009be760323d., hostname=bd6592c37bb2,40087,1732486423713, seqNum=2 , the old value is region=TestAcidGuarantees,,1732486556329.744bbec6934be304e01b009be760323d., hostname=bd6592c37bb2,40087,1732486423713, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=744bbec6934be304e01b009be760323d, server=bd6592c37bb2,40087,1732486423713 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T22:16:14,916 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732486556329.744bbec6934be304e01b009be760323d., hostname=bd6592c37bb2,40087,1732486423713, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=744bbec6934be304e01b009be760323d, server=bd6592c37bb2,40087,1732486423713 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T22:16:14,916 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732486556329.744bbec6934be304e01b009be760323d., hostname=bd6592c37bb2,40087,1732486423713, seqNum=2 because the exception is null or not the one we care about 2024-11-24T22:16:14,920 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=40087 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=136 2024-11-24T22:16:14,921 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bd6592c37bb2:0-0 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732486556329.744bbec6934be304e01b009be760323d. 2024-11-24T22:16:14,921 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bd6592c37bb2:0-0 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732486556329.744bbec6934be304e01b009be760323d. as already flushing 2024-11-24T22:16:14,921 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bd6592c37bb2:0-0 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732486556329.744bbec6934be304e01b009be760323d. 2024-11-24T22:16:14,921 ERROR [RS_FLUSH_OPERATIONS-regionserver/bd6592c37bb2:0-0 {event_type=RS_FLUSH_REGIONS, pid=136}] handler.RSProcedureHandler(58): pid=136 java.io.IOException: Unable to complete flush {ENCODED => 744bbec6934be304e01b009be760323d, NAME => 'TestAcidGuarantees,,1732486556329.744bbec6934be304e01b009be760323d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T22:16:14,921 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bd6592c37bb2:0-0 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=136 java.io.IOException: Unable to complete flush {ENCODED => 744bbec6934be304e01b009be760323d, NAME => 'TestAcidGuarantees,,1732486556329.744bbec6934be304e01b009be760323d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T22:16:14,922 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37701 {}] master.HMaster(4177): Remote procedure failed, pid=136 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 744bbec6934be304e01b009be760323d, NAME => 'TestAcidGuarantees,,1732486556329.744bbec6934be304e01b009be760323d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 744bbec6934be304e01b009be760323d, NAME => 'TestAcidGuarantees,,1732486556329.744bbec6934be304e01b009be760323d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T22:16:14,931 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40087 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=744bbec6934be304e01b009be760323d, server=bd6592c37bb2,40087,1732486423713 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-24T22:16:14,931 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40087 {}] ipc.CallRunner(138): callId: 166 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46792 deadline: 1732486634921, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=744bbec6934be304e01b009be760323d, server=bd6592c37bb2,40087,1732486423713 2024-11-24T22:16:14,932 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732486556329.744bbec6934be304e01b009be760323d., hostname=bd6592c37bb2,40087,1732486423713, seqNum=2 , the old value is region=TestAcidGuarantees,,1732486556329.744bbec6934be304e01b009be760323d., hostname=bd6592c37bb2,40087,1732486423713, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=744bbec6934be304e01b009be760323d, server=bd6592c37bb2,40087,1732486423713 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T22:16:14,932 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732486556329.744bbec6934be304e01b009be760323d., hostname=bd6592c37bb2,40087,1732486423713, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=744bbec6934be304e01b009be760323d, server=bd6592c37bb2,40087,1732486423713 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T22:16:14,932 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732486556329.744bbec6934be304e01b009be760323d., hostname=bd6592c37bb2,40087,1732486423713, seqNum=2 because the exception is null or not the one we care about 2024-11-24T22:16:14,983 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40087 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=744bbec6934be304e01b009be760323d, server=bd6592c37bb2,40087,1732486423713 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-24T22:16:14,983 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40087 {}] ipc.CallRunner(138): callId: 69 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46820 deadline: 1732486634970, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=744bbec6934be304e01b009be760323d, server=bd6592c37bb2,40087,1732486423713 2024-11-24T22:16:14,983 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40087 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=744bbec6934be304e01b009be760323d, server=bd6592c37bb2,40087,1732486423713 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-24T22:16:14,984 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40087 {}] ipc.CallRunner(138): callId: 77 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46824 deadline: 1732486634971, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=744bbec6934be304e01b009be760323d, server=bd6592c37bb2,40087,1732486423713 2024-11-24T22:16:14,984 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40087 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=744bbec6934be304e01b009be760323d, server=bd6592c37bb2,40087,1732486423713 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-24T22:16:14,984 WARN [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRpcRetryingCaller(177): Call to bd6592c37bb2,40087,1732486423713 for 'test_row_0' in 744bbec6934be304e01b009be760323d of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2232 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=744bbec6934be304e01b009be760323d, server=bd6592c37bb2,40087,1732486423713 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor44.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=744bbec6934be304e01b009be760323d, server=bd6592c37bb2,40087,1732486423713 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-24T22:16:14,984 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40087 {}] ipc.CallRunner(138): callId: 68 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46816 deadline: 1732486634971, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=744bbec6934be304e01b009be760323d, server=bd6592c37bb2,40087,1732486423713 2024-11-24T22:16:14,984 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732486556329.744bbec6934be304e01b009be760323d., hostname=bd6592c37bb2,40087,1732486423713, seqNum=2 , the old value is region=TestAcidGuarantees,,1732486556329.744bbec6934be304e01b009be760323d., hostname=bd6592c37bb2,40087,1732486423713, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=744bbec6934be304e01b009be760323d, server=bd6592c37bb2,40087,1732486423713 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T22:16:14,984 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732486556329.744bbec6934be304e01b009be760323d., hostname=bd6592c37bb2,40087,1732486423713, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=744bbec6934be304e01b009be760323d, server=bd6592c37bb2,40087,1732486423713 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T22:16:14,984 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732486556329.744bbec6934be304e01b009be760323d., hostname=bd6592c37bb2,40087,1732486423713, seqNum=2 because the exception is null or not the one we care about 2024-11-24T22:16:14,984 WARN [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRpcRetryingCaller(177): Call to bd6592c37bb2,40087,1732486423713 for 'test_row_2' in 744bbec6934be304e01b009be760323d of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2230 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=744bbec6934be304e01b009be760323d, server=bd6592c37bb2,40087,1732486423713 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor44.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=744bbec6934be304e01b009be760323d, server=bd6592c37bb2,40087,1732486423713 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-24T22:16:14,985 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732486556329.744bbec6934be304e01b009be760323d., hostname=bd6592c37bb2,40087,1732486423713, seqNum=2 , the old value is region=TestAcidGuarantees,,1732486556329.744bbec6934be304e01b009be760323d., hostname=bd6592c37bb2,40087,1732486423713, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=744bbec6934be304e01b009be760323d, server=bd6592c37bb2,40087,1732486423713 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T22:16:14,985 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732486556329.744bbec6934be304e01b009be760323d., hostname=bd6592c37bb2,40087,1732486423713, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=744bbec6934be304e01b009be760323d, server=bd6592c37bb2,40087,1732486423713 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T22:16:14,985 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732486556329.744bbec6934be304e01b009be760323d., hostname=bd6592c37bb2,40087,1732486423713, seqNum=2 because the exception is null or not the one we care about 2024-11-24T22:16:14,984 WARN [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRpcRetryingCaller(177): Call to bd6592c37bb2,40087,1732486423713 for 'test_row_2' in 744bbec6934be304e01b009be760323d of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2230 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=744bbec6934be304e01b009be760323d, server=bd6592c37bb2,40087,1732486423713 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor44.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=744bbec6934be304e01b009be760323d, server=bd6592c37bb2,40087,1732486423713 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-24T22:16:14,985 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732486556329.744bbec6934be304e01b009be760323d., hostname=bd6592c37bb2,40087,1732486423713, seqNum=2 , the old value is region=TestAcidGuarantees,,1732486556329.744bbec6934be304e01b009be760323d., hostname=bd6592c37bb2,40087,1732486423713, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=744bbec6934be304e01b009be760323d, server=bd6592c37bb2,40087,1732486423713 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T22:16:14,985 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732486556329.744bbec6934be304e01b009be760323d., hostname=bd6592c37bb2,40087,1732486423713, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=744bbec6934be304e01b009be760323d, server=bd6592c37bb2,40087,1732486423713 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T22:16:14,985 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732486556329.744bbec6934be304e01b009be760323d., hostname=bd6592c37bb2,40087,1732486423713, seqNum=2 because the exception is null or not the one we care about 2024-11-24T22:16:15,074 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=40087 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=136 2024-11-24T22:16:15,074 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bd6592c37bb2:0-1 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732486556329.744bbec6934be304e01b009be760323d. 2024-11-24T22:16:15,074 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bd6592c37bb2:0-1 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732486556329.744bbec6934be304e01b009be760323d. as already flushing 2024-11-24T22:16:15,074 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bd6592c37bb2:0-1 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732486556329.744bbec6934be304e01b009be760323d. 2024-11-24T22:16:15,074 ERROR [RS_FLUSH_OPERATIONS-regionserver/bd6592c37bb2:0-1 {event_type=RS_FLUSH_REGIONS, pid=136}] handler.RSProcedureHandler(58): pid=136 java.io.IOException: Unable to complete flush {ENCODED => 744bbec6934be304e01b009be760323d, NAME => 'TestAcidGuarantees,,1732486556329.744bbec6934be304e01b009be760323d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T22:16:15,075 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bd6592c37bb2:0-1 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=136 java.io.IOException: Unable to complete flush {ENCODED => 744bbec6934be304e01b009be760323d, NAME => 'TestAcidGuarantees,,1732486556329.744bbec6934be304e01b009be760323d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T22:16:15,075 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37701 {}] master.HMaster(4177): Remote procedure failed, pid=136 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 744bbec6934be304e01b009be760323d, NAME => 'TestAcidGuarantees,,1732486556329.744bbec6934be304e01b009be760323d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 744bbec6934be304e01b009be760323d, NAME => 'TestAcidGuarantees,,1732486556329.744bbec6934be304e01b009be760323d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T22:16:15,081 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37701 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=135 2024-11-24T22:16:15,132 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40087 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=744bbec6934be304e01b009be760323d, server=bd6592c37bb2,40087,1732486423713 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-24T22:16:15,132 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40087 {}] ipc.CallRunner(138): callId: 91 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46842 deadline: 1732486635121, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=744bbec6934be304e01b009be760323d, server=bd6592c37bb2,40087,1732486423713 2024-11-24T22:16:15,132 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732486556329.744bbec6934be304e01b009be760323d., hostname=bd6592c37bb2,40087,1732486423713, seqNum=2 , the old value is region=TestAcidGuarantees,,1732486556329.744bbec6934be304e01b009be760323d., hostname=bd6592c37bb2,40087,1732486423713, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=744bbec6934be304e01b009be760323d, server=bd6592c37bb2,40087,1732486423713 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T22:16:15,132 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732486556329.744bbec6934be304e01b009be760323d., hostname=bd6592c37bb2,40087,1732486423713, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=744bbec6934be304e01b009be760323d, server=bd6592c37bb2,40087,1732486423713 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T22:16:15,132 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732486556329.744bbec6934be304e01b009be760323d., hostname=bd6592c37bb2,40087,1732486423713, seqNum=2 because the exception is null or not the one we care about 2024-11-24T22:16:15,148 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=31.31 KB at sequenceid=344 (bloomFilter=true), to=hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/744bbec6934be304e01b009be760323d/.tmp/A/8007d32c8dc5404ea79a5762cb35d83d 2024-11-24T22:16:15,150 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40087 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=744bbec6934be304e01b009be760323d, server=bd6592c37bb2,40087,1732486423713 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-24T22:16:15,151 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40087 {}] ipc.CallRunner(138): callId: 167 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46792 deadline: 1732486635140, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=744bbec6934be304e01b009be760323d, server=bd6592c37bb2,40087,1732486423713 2024-11-24T22:16:15,151 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732486556329.744bbec6934be304e01b009be760323d., hostname=bd6592c37bb2,40087,1732486423713, seqNum=2 , the old value is region=TestAcidGuarantees,,1732486556329.744bbec6934be304e01b009be760323d., hostname=bd6592c37bb2,40087,1732486423713, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=744bbec6934be304e01b009be760323d, server=bd6592c37bb2,40087,1732486423713 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T22:16:15,151 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732486556329.744bbec6934be304e01b009be760323d., hostname=bd6592c37bb2,40087,1732486423713, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=744bbec6934be304e01b009be760323d, server=bd6592c37bb2,40087,1732486423713 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T22:16:15,151 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732486556329.744bbec6934be304e01b009be760323d., hostname=bd6592c37bb2,40087,1732486423713, seqNum=2 because the exception is null or not the one we care about 2024-11-24T22:16:15,152 DEBUG [RS:0;bd6592c37bb2:40087-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/744bbec6934be304e01b009be760323d/.tmp/A/4cbb2b50c7d74794986c166190e79fdb as hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/744bbec6934be304e01b009be760323d/A/4cbb2b50c7d74794986c166190e79fdb 2024-11-24T22:16:15,156 INFO [RS:0;bd6592c37bb2:40087-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 744bbec6934be304e01b009be760323d/A of 744bbec6934be304e01b009be760323d into 4cbb2b50c7d74794986c166190e79fdb(size=12.8 K), total size for store is 12.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-24T22:16:15,156 DEBUG [RS:0;bd6592c37bb2:40087-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 744bbec6934be304e01b009be760323d: 2024-11-24T22:16:15,156 INFO [RS:0;bd6592c37bb2:40087-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732486556329.744bbec6934be304e01b009be760323d., storeName=744bbec6934be304e01b009be760323d/A, priority=13, startTime=1732486574706; duration=0sec 2024-11-24T22:16:15,157 DEBUG [RS:0;bd6592c37bb2:40087-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-24T22:16:15,157 DEBUG [RS:0;bd6592c37bb2:40087-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 744bbec6934be304e01b009be760323d:A 2024-11-24T22:16:15,158 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/744bbec6934be304e01b009be760323d/.tmp/B/6fe913fa79a24f6fae7b349083870039 is 50, key is test_row_0/B:col10/1732486574698/Put/seqid=0 2024-11-24T22:16:15,178 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45929 is added to blk_1073742354_1530 (size=12301) 2024-11-24T22:16:15,191 DEBUG [RS:0;bd6592c37bb2:40087-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/744bbec6934be304e01b009be760323d/.tmp/C/18cfa9a432cd4a559609140d0121d730 as hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/744bbec6934be304e01b009be760323d/C/18cfa9a432cd4a559609140d0121d730 2024-11-24T22:16:15,196 INFO [RS:0;bd6592c37bb2:40087-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 744bbec6934be304e01b009be760323d/C of 744bbec6934be304e01b009be760323d into 18cfa9a432cd4a559609140d0121d730(size=12.8 K), total size for store is 12.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-24T22:16:15,196 DEBUG [RS:0;bd6592c37bb2:40087-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 744bbec6934be304e01b009be760323d: 2024-11-24T22:16:15,196 INFO [RS:0;bd6592c37bb2:40087-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732486556329.744bbec6934be304e01b009be760323d., storeName=744bbec6934be304e01b009be760323d/C, priority=13, startTime=1732486574707; duration=0sec 2024-11-24T22:16:15,196 DEBUG [RS:0;bd6592c37bb2:40087-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-24T22:16:15,196 DEBUG [RS:0;bd6592c37bb2:40087-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 744bbec6934be304e01b009be760323d:C 2024-11-24T22:16:15,227 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=40087 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=136 2024-11-24T22:16:15,227 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bd6592c37bb2:0-2 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732486556329.744bbec6934be304e01b009be760323d. 2024-11-24T22:16:15,227 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bd6592c37bb2:0-2 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732486556329.744bbec6934be304e01b009be760323d. as already flushing 2024-11-24T22:16:15,228 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bd6592c37bb2:0-2 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732486556329.744bbec6934be304e01b009be760323d. 2024-11-24T22:16:15,228 ERROR [RS_FLUSH_OPERATIONS-regionserver/bd6592c37bb2:0-2 {event_type=RS_FLUSH_REGIONS, pid=136}] handler.RSProcedureHandler(58): pid=136 java.io.IOException: Unable to complete flush {ENCODED => 744bbec6934be304e01b009be760323d, NAME => 'TestAcidGuarantees,,1732486556329.744bbec6934be304e01b009be760323d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T22:16:15,228 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bd6592c37bb2:0-2 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=136 java.io.IOException: Unable to complete flush {ENCODED => 744bbec6934be304e01b009be760323d, NAME => 'TestAcidGuarantees,,1732486556329.744bbec6934be304e01b009be760323d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T22:16:15,228 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37701 {}] master.HMaster(4177): Remote procedure failed, pid=136 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 744bbec6934be304e01b009be760323d, NAME => 'TestAcidGuarantees,,1732486556329.744bbec6934be304e01b009be760323d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 744bbec6934be304e01b009be760323d, NAME => 'TestAcidGuarantees,,1732486556329.744bbec6934be304e01b009be760323d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T22:16:15,380 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=40087 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=136 2024-11-24T22:16:15,381 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bd6592c37bb2:0-0 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732486556329.744bbec6934be304e01b009be760323d. 2024-11-24T22:16:15,381 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bd6592c37bb2:0-0 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732486556329.744bbec6934be304e01b009be760323d. as already flushing 2024-11-24T22:16:15,381 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bd6592c37bb2:0-0 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732486556329.744bbec6934be304e01b009be760323d. 2024-11-24T22:16:15,381 ERROR [RS_FLUSH_OPERATIONS-regionserver/bd6592c37bb2:0-0 {event_type=RS_FLUSH_REGIONS, pid=136}] handler.RSProcedureHandler(58): pid=136 java.io.IOException: Unable to complete flush {ENCODED => 744bbec6934be304e01b009be760323d, NAME => 'TestAcidGuarantees,,1732486556329.744bbec6934be304e01b009be760323d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T22:16:15,381 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bd6592c37bb2:0-0 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=136 java.io.IOException: Unable to complete flush {ENCODED => 744bbec6934be304e01b009be760323d, NAME => 'TestAcidGuarantees,,1732486556329.744bbec6934be304e01b009be760323d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T22:16:15,382 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37701 {}] master.HMaster(4177): Remote procedure failed, pid=136 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 744bbec6934be304e01b009be760323d, NAME => 'TestAcidGuarantees,,1732486556329.744bbec6934be304e01b009be760323d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 744bbec6934be304e01b009be760323d, NAME => 'TestAcidGuarantees,,1732486556329.744bbec6934be304e01b009be760323d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T22:16:15,391 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37701 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=135 2024-11-24T22:16:15,453 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40087 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=744bbec6934be304e01b009be760323d, server=bd6592c37bb2,40087,1732486423713 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-24T22:16:15,453 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40087 {}] ipc.CallRunner(138): callId: 92 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46842 deadline: 1732486635440, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=744bbec6934be304e01b009be760323d, server=bd6592c37bb2,40087,1732486423713 2024-11-24T22:16:15,454 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732486556329.744bbec6934be304e01b009be760323d., hostname=bd6592c37bb2,40087,1732486423713, seqNum=2 , the old value is region=TestAcidGuarantees,,1732486556329.744bbec6934be304e01b009be760323d., hostname=bd6592c37bb2,40087,1732486423713, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=744bbec6934be304e01b009be760323d, server=bd6592c37bb2,40087,1732486423713 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T22:16:15,454 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732486556329.744bbec6934be304e01b009be760323d., hostname=bd6592c37bb2,40087,1732486423713, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=744bbec6934be304e01b009be760323d, server=bd6592c37bb2,40087,1732486423713 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T22:16:15,454 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732486556329.744bbec6934be304e01b009be760323d., hostname=bd6592c37bb2,40087,1732486423713, seqNum=2 because the exception is null or not the one we care about 2024-11-24T22:16:15,467 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40087 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=744bbec6934be304e01b009be760323d, server=bd6592c37bb2,40087,1732486423713 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-24T22:16:15,467 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40087 {}] ipc.CallRunner(138): callId: 168 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46792 deadline: 1732486635461, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=744bbec6934be304e01b009be760323d, server=bd6592c37bb2,40087,1732486423713 2024-11-24T22:16:15,467 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732486556329.744bbec6934be304e01b009be760323d., hostname=bd6592c37bb2,40087,1732486423713, seqNum=2 , the old value is region=TestAcidGuarantees,,1732486556329.744bbec6934be304e01b009be760323d., hostname=bd6592c37bb2,40087,1732486423713, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=744bbec6934be304e01b009be760323d, server=bd6592c37bb2,40087,1732486423713 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T22:16:15,467 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732486556329.744bbec6934be304e01b009be760323d., hostname=bd6592c37bb2,40087,1732486423713, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=744bbec6934be304e01b009be760323d, server=bd6592c37bb2,40087,1732486423713 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T22:16:15,467 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732486556329.744bbec6934be304e01b009be760323d., hostname=bd6592c37bb2,40087,1732486423713, seqNum=2 because the exception is null or not the one we care about 2024-11-24T22:16:15,534 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=40087 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=136 2024-11-24T22:16:15,534 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bd6592c37bb2:0-1 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732486556329.744bbec6934be304e01b009be760323d. 2024-11-24T22:16:15,534 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bd6592c37bb2:0-1 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732486556329.744bbec6934be304e01b009be760323d. as already flushing 2024-11-24T22:16:15,534 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bd6592c37bb2:0-1 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732486556329.744bbec6934be304e01b009be760323d. 2024-11-24T22:16:15,534 ERROR [RS_FLUSH_OPERATIONS-regionserver/bd6592c37bb2:0-1 {event_type=RS_FLUSH_REGIONS, pid=136}] handler.RSProcedureHandler(58): pid=136 java.io.IOException: Unable to complete flush {ENCODED => 744bbec6934be304e01b009be760323d, NAME => 'TestAcidGuarantees,,1732486556329.744bbec6934be304e01b009be760323d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T22:16:15,534 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bd6592c37bb2:0-1 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=136 java.io.IOException: Unable to complete flush {ENCODED => 744bbec6934be304e01b009be760323d, NAME => 'TestAcidGuarantees,,1732486556329.744bbec6934be304e01b009be760323d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T22:16:15,535 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37701 {}] master.HMaster(4177): Remote procedure failed, pid=136 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 744bbec6934be304e01b009be760323d, NAME => 'TestAcidGuarantees,,1732486556329.744bbec6934be304e01b009be760323d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 744bbec6934be304e01b009be760323d, NAME => 'TestAcidGuarantees,,1732486556329.744bbec6934be304e01b009be760323d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T22:16:15,578 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=31.31 KB at sequenceid=344 (bloomFilter=true), to=hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/744bbec6934be304e01b009be760323d/.tmp/B/6fe913fa79a24f6fae7b349083870039 2024-11-24T22:16:15,590 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/744bbec6934be304e01b009be760323d/.tmp/C/5d83f39257e349a0b43d8f3dd1fdf19b is 50, key is test_row_0/C:col10/1732486574698/Put/seqid=0 2024-11-24T22:16:15,638 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45929 is added to blk_1073742355_1531 (size=12301) 2024-11-24T22:16:15,638 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=31.31 KB at sequenceid=344 (bloomFilter=true), to=hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/744bbec6934be304e01b009be760323d/.tmp/C/5d83f39257e349a0b43d8f3dd1fdf19b 2024-11-24T22:16:15,646 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/744bbec6934be304e01b009be760323d/.tmp/A/8007d32c8dc5404ea79a5762cb35d83d as hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/744bbec6934be304e01b009be760323d/A/8007d32c8dc5404ea79a5762cb35d83d 2024-11-24T22:16:15,649 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/744bbec6934be304e01b009be760323d/A/8007d32c8dc5404ea79a5762cb35d83d, entries=150, sequenceid=344, filesize=12.0 K 2024-11-24T22:16:15,652 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/744bbec6934be304e01b009be760323d/.tmp/B/6fe913fa79a24f6fae7b349083870039 as hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/744bbec6934be304e01b009be760323d/B/6fe913fa79a24f6fae7b349083870039 2024-11-24T22:16:15,662 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/744bbec6934be304e01b009be760323d/B/6fe913fa79a24f6fae7b349083870039, entries=150, sequenceid=344, filesize=12.0 K 2024-11-24T22:16:15,663 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/744bbec6934be304e01b009be760323d/.tmp/C/5d83f39257e349a0b43d8f3dd1fdf19b as hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/744bbec6934be304e01b009be760323d/C/5d83f39257e349a0b43d8f3dd1fdf19b 2024-11-24T22:16:15,672 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/744bbec6934be304e01b009be760323d/C/5d83f39257e349a0b43d8f3dd1fdf19b, entries=150, sequenceid=344, filesize=12.0 K 2024-11-24T22:16:15,673 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~93.93 KB/96180, heapSize ~279.61 KB/286320, currentSize=80.51 KB/82440 for 744bbec6934be304e01b009be760323d in 961ms, sequenceid=344, compaction requested=false 2024-11-24T22:16:15,673 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 744bbec6934be304e01b009be760323d: 2024-11-24T22:16:15,686 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=40087 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=136 2024-11-24T22:16:15,686 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bd6592c37bb2:0-2 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732486556329.744bbec6934be304e01b009be760323d. 2024-11-24T22:16:15,687 INFO [RS_FLUSH_OPERATIONS-regionserver/bd6592c37bb2:0-2 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.HRegion(2902): Flushing 744bbec6934be304e01b009be760323d 3/3 column families, dataSize=80.51 KB heapSize=239.81 KB 2024-11-24T22:16:15,687 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bd6592c37bb2:0-2 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 744bbec6934be304e01b009be760323d, store=A 2024-11-24T22:16:15,687 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bd6592c37bb2:0-2 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-24T22:16:15,687 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bd6592c37bb2:0-2 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 744bbec6934be304e01b009be760323d, store=B 2024-11-24T22:16:15,687 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bd6592c37bb2:0-2 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-24T22:16:15,687 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bd6592c37bb2:0-2 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 744bbec6934be304e01b009be760323d, store=C 2024-11-24T22:16:15,687 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bd6592c37bb2:0-2 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-24T22:16:15,711 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bd6592c37bb2:0-2 {event_type=RS_FLUSH_REGIONS, pid=136}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/744bbec6934be304e01b009be760323d/.tmp/A/ed730b6e50aa4416aa0742d68965b490 is 50, key is test_row_0/A:col10/1732486574760/Put/seqid=0 2024-11-24T22:16:15,726 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45929 is added to blk_1073742356_1532 (size=12301) 2024-11-24T22:16:15,900 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37701 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=135 2024-11-24T22:16:15,964 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732486556329.744bbec6934be304e01b009be760323d. as already flushing 2024-11-24T22:16:15,965 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40087 {}] regionserver.HRegion(8855): Flush requested on 744bbec6934be304e01b009be760323d 2024-11-24T22:16:16,069 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40087 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=744bbec6934be304e01b009be760323d, server=bd6592c37bb2,40087,1732486423713 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-24T22:16:16,069 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40087 {}] ipc.CallRunner(138): callId: 174 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46792 deadline: 1732486636055, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=744bbec6934be304e01b009be760323d, server=bd6592c37bb2,40087,1732486423713 2024-11-24T22:16:16,070 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732486556329.744bbec6934be304e01b009be760323d., hostname=bd6592c37bb2,40087,1732486423713, seqNum=2 , the old value is region=TestAcidGuarantees,,1732486556329.744bbec6934be304e01b009be760323d., hostname=bd6592c37bb2,40087,1732486423713, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=744bbec6934be304e01b009be760323d, server=bd6592c37bb2,40087,1732486423713 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T22:16:16,070 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732486556329.744bbec6934be304e01b009be760323d., hostname=bd6592c37bb2,40087,1732486423713, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=744bbec6934be304e01b009be760323d, server=bd6592c37bb2,40087,1732486423713 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T22:16:16,070 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732486556329.744bbec6934be304e01b009be760323d., hostname=bd6592c37bb2,40087,1732486423713, seqNum=2 because the exception is null or not the one we care about 2024-11-24T22:16:16,073 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40087 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=744bbec6934be304e01b009be760323d, server=bd6592c37bb2,40087,1732486423713 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-24T22:16:16,073 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40087 {}] ipc.CallRunner(138): callId: 102 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46842 deadline: 1732486636067, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=744bbec6934be304e01b009be760323d, server=bd6592c37bb2,40087,1732486423713 2024-11-24T22:16:16,073 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732486556329.744bbec6934be304e01b009be760323d., hostname=bd6592c37bb2,40087,1732486423713, seqNum=2 , the old value is region=TestAcidGuarantees,,1732486556329.744bbec6934be304e01b009be760323d., hostname=bd6592c37bb2,40087,1732486423713, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=744bbec6934be304e01b009be760323d, server=bd6592c37bb2,40087,1732486423713 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T22:16:16,073 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732486556329.744bbec6934be304e01b009be760323d., hostname=bd6592c37bb2,40087,1732486423713, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=744bbec6934be304e01b009be760323d, server=bd6592c37bb2,40087,1732486423713 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T22:16:16,073 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732486556329.744bbec6934be304e01b009be760323d., hostname=bd6592c37bb2,40087,1732486423713, seqNum=2 because the exception is null or not the one we care about 2024-11-24T22:16:16,126 INFO [RS_FLUSH_OPERATIONS-regionserver/bd6592c37bb2:0-2 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=26.84 KB at sequenceid=362 (bloomFilter=true), to=hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/744bbec6934be304e01b009be760323d/.tmp/A/ed730b6e50aa4416aa0742d68965b490 2024-11-24T22:16:16,137 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bd6592c37bb2:0-2 {event_type=RS_FLUSH_REGIONS, pid=136}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/744bbec6934be304e01b009be760323d/.tmp/B/a25b2da6745546899c020b13033d9224 is 50, key is test_row_0/B:col10/1732486574760/Put/seqid=0 2024-11-24T22:16:16,153 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45929 is added to blk_1073742357_1533 (size=12301) 2024-11-24T22:16:16,159 INFO [RS_FLUSH_OPERATIONS-regionserver/bd6592c37bb2:0-2 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=26.84 KB at sequenceid=362 (bloomFilter=true), to=hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/744bbec6934be304e01b009be760323d/.tmp/B/a25b2da6745546899c020b13033d9224 2024-11-24T22:16:16,168 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bd6592c37bb2:0-2 {event_type=RS_FLUSH_REGIONS, pid=136}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/744bbec6934be304e01b009be760323d/.tmp/C/a23bbfecd00441eab0a749d4ed2a5cd5 is 50, key is test_row_0/C:col10/1732486574760/Put/seqid=0 2024-11-24T22:16:16,174 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45929 is added to blk_1073742358_1534 (size=12301) 2024-11-24T22:16:16,174 INFO [RS_FLUSH_OPERATIONS-regionserver/bd6592c37bb2:0-2 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=26.84 KB at sequenceid=362 (bloomFilter=true), to=hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/744bbec6934be304e01b009be760323d/.tmp/C/a23bbfecd00441eab0a749d4ed2a5cd5 2024-11-24T22:16:16,180 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bd6592c37bb2:0-2 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/744bbec6934be304e01b009be760323d/.tmp/A/ed730b6e50aa4416aa0742d68965b490 as hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/744bbec6934be304e01b009be760323d/A/ed730b6e50aa4416aa0742d68965b490 2024-11-24T22:16:16,184 INFO [RS_FLUSH_OPERATIONS-regionserver/bd6592c37bb2:0-2 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/744bbec6934be304e01b009be760323d/A/ed730b6e50aa4416aa0742d68965b490, entries=150, sequenceid=362, filesize=12.0 K 2024-11-24T22:16:16,184 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bd6592c37bb2:0-2 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/744bbec6934be304e01b009be760323d/.tmp/B/a25b2da6745546899c020b13033d9224 as hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/744bbec6934be304e01b009be760323d/B/a25b2da6745546899c020b13033d9224 2024-11-24T22:16:16,189 INFO [RS_FLUSH_OPERATIONS-regionserver/bd6592c37bb2:0-2 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/744bbec6934be304e01b009be760323d/B/a25b2da6745546899c020b13033d9224, entries=150, sequenceid=362, filesize=12.0 K 2024-11-24T22:16:16,190 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40087 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=744bbec6934be304e01b009be760323d, server=bd6592c37bb2,40087,1732486423713 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-24T22:16:16,190 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40087 {}] ipc.CallRunner(138): callId: 175 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46792 deadline: 1732486636181, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=744bbec6934be304e01b009be760323d, server=bd6592c37bb2,40087,1732486423713 2024-11-24T22:16:16,191 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732486556329.744bbec6934be304e01b009be760323d., hostname=bd6592c37bb2,40087,1732486423713, seqNum=2 , the old value is region=TestAcidGuarantees,,1732486556329.744bbec6934be304e01b009be760323d., hostname=bd6592c37bb2,40087,1732486423713, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=744bbec6934be304e01b009be760323d, server=bd6592c37bb2,40087,1732486423713 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T22:16:16,191 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40087 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=744bbec6934be304e01b009be760323d, server=bd6592c37bb2,40087,1732486423713 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-24T22:16:16,191 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732486556329.744bbec6934be304e01b009be760323d., hostname=bd6592c37bb2,40087,1732486423713, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=744bbec6934be304e01b009be760323d, server=bd6592c37bb2,40087,1732486423713 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T22:16:16,191 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732486556329.744bbec6934be304e01b009be760323d., hostname=bd6592c37bb2,40087,1732486423713, seqNum=2 because the exception is null or not the one we care about 2024-11-24T22:16:16,191 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40087 {}] ipc.CallRunner(138): callId: 103 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46842 deadline: 1732486636181, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=744bbec6934be304e01b009be760323d, server=bd6592c37bb2,40087,1732486423713 2024-11-24T22:16:16,191 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732486556329.744bbec6934be304e01b009be760323d., hostname=bd6592c37bb2,40087,1732486423713, seqNum=2 , the old value is region=TestAcidGuarantees,,1732486556329.744bbec6934be304e01b009be760323d., hostname=bd6592c37bb2,40087,1732486423713, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=744bbec6934be304e01b009be760323d, server=bd6592c37bb2,40087,1732486423713 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T22:16:16,191 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732486556329.744bbec6934be304e01b009be760323d., hostname=bd6592c37bb2,40087,1732486423713, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=744bbec6934be304e01b009be760323d, server=bd6592c37bb2,40087,1732486423713 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T22:16:16,191 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732486556329.744bbec6934be304e01b009be760323d., hostname=bd6592c37bb2,40087,1732486423713, seqNum=2 because the exception is null or not the one we care about 2024-11-24T22:16:16,195 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bd6592c37bb2:0-2 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/744bbec6934be304e01b009be760323d/.tmp/C/a23bbfecd00441eab0a749d4ed2a5cd5 as hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/744bbec6934be304e01b009be760323d/C/a23bbfecd00441eab0a749d4ed2a5cd5 2024-11-24T22:16:16,198 INFO [RS_FLUSH_OPERATIONS-regionserver/bd6592c37bb2:0-2 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/744bbec6934be304e01b009be760323d/C/a23bbfecd00441eab0a749d4ed2a5cd5, entries=150, sequenceid=362, filesize=12.0 K 2024-11-24T22:16:16,199 INFO [RS_FLUSH_OPERATIONS-regionserver/bd6592c37bb2:0-2 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.HRegion(3140): Finished flush of dataSize ~80.51 KB/82440, heapSize ~239.77 KB/245520, currentSize=93.93 KB/96180 for 744bbec6934be304e01b009be760323d in 513ms, sequenceid=362, compaction requested=true 2024-11-24T22:16:16,199 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bd6592c37bb2:0-2 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.HRegion(2603): Flush status journal for 744bbec6934be304e01b009be760323d: 2024-11-24T22:16:16,199 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bd6592c37bb2:0-2 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732486556329.744bbec6934be304e01b009be760323d. 2024-11-24T22:16:16,199 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bd6592c37bb2:0-2 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=136 2024-11-24T22:16:16,200 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37701 {}] master.HMaster(4169): Remote procedure done, pid=136 2024-11-24T22:16:16,202 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=136, resume processing ppid=135 2024-11-24T22:16:16,203 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=136, ppid=135, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.4340 sec 2024-11-24T22:16:16,204 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=135, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=135, table=TestAcidGuarantees in 1.4390 sec 2024-11-24T22:16:16,408 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40087 {}] regionserver.HRegion(8855): Flush requested on 744bbec6934be304e01b009be760323d 2024-11-24T22:16:16,408 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 744bbec6934be304e01b009be760323d 3/3 column families, dataSize=100.63 KB heapSize=299.58 KB 2024-11-24T22:16:16,408 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 744bbec6934be304e01b009be760323d, store=A 2024-11-24T22:16:16,408 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-24T22:16:16,408 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 744bbec6934be304e01b009be760323d, store=B 2024-11-24T22:16:16,408 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-24T22:16:16,408 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 744bbec6934be304e01b009be760323d, store=C 2024-11-24T22:16:16,408 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-24T22:16:16,413 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/744bbec6934be304e01b009be760323d/.tmp/A/513165e189654d61a14d6ba01582bce3 is 50, key is test_row_0/A:col10/1732486576053/Put/seqid=0 2024-11-24T22:16:16,420 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45929 is added to blk_1073742359_1535 (size=14741) 2024-11-24T22:16:16,421 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=33.54 KB at sequenceid=380 (bloomFilter=true), to=hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/744bbec6934be304e01b009be760323d/.tmp/A/513165e189654d61a14d6ba01582bce3 2024-11-24T22:16:16,428 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/744bbec6934be304e01b009be760323d/.tmp/B/09209f77165d4e7f9f6c4c1ed40f9136 is 50, key is test_row_0/B:col10/1732486576053/Put/seqid=0 2024-11-24T22:16:16,431 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45929 is added to blk_1073742360_1536 (size=12301) 2024-11-24T22:16:16,431 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=33.54 KB at sequenceid=380 (bloomFilter=true), to=hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/744bbec6934be304e01b009be760323d/.tmp/B/09209f77165d4e7f9f6c4c1ed40f9136 2024-11-24T22:16:16,439 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/744bbec6934be304e01b009be760323d/.tmp/C/d2585fa8933643ddb5981789762dddba is 50, key is test_row_0/C:col10/1732486576053/Put/seqid=0 2024-11-24T22:16:16,445 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45929 is added to blk_1073742361_1537 (size=12301) 2024-11-24T22:16:16,446 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=33.54 KB at sequenceid=380 (bloomFilter=true), to=hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/744bbec6934be304e01b009be760323d/.tmp/C/d2585fa8933643ddb5981789762dddba 2024-11-24T22:16:16,449 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/744bbec6934be304e01b009be760323d/.tmp/A/513165e189654d61a14d6ba01582bce3 as hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/744bbec6934be304e01b009be760323d/A/513165e189654d61a14d6ba01582bce3 2024-11-24T22:16:16,451 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/744bbec6934be304e01b009be760323d/A/513165e189654d61a14d6ba01582bce3, entries=200, sequenceid=380, filesize=14.4 K 2024-11-24T22:16:16,455 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/744bbec6934be304e01b009be760323d/.tmp/B/09209f77165d4e7f9f6c4c1ed40f9136 as hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/744bbec6934be304e01b009be760323d/B/09209f77165d4e7f9f6c4c1ed40f9136 2024-11-24T22:16:16,457 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/744bbec6934be304e01b009be760323d/B/09209f77165d4e7f9f6c4c1ed40f9136, entries=150, sequenceid=380, filesize=12.0 K 2024-11-24T22:16:16,458 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/744bbec6934be304e01b009be760323d/.tmp/C/d2585fa8933643ddb5981789762dddba as hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/744bbec6934be304e01b009be760323d/C/d2585fa8933643ddb5981789762dddba 2024-11-24T22:16:16,461 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/744bbec6934be304e01b009be760323d/C/d2585fa8933643ddb5981789762dddba, entries=150, sequenceid=380, filesize=12.0 K 2024-11-24T22:16:16,462 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~100.63 KB/103050, heapSize ~299.53 KB/306720, currentSize=67.09 KB/68700 for 744bbec6934be304e01b009be760323d in 54ms, sequenceid=380, compaction requested=true 2024-11-24T22:16:16,462 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 744bbec6934be304e01b009be760323d: 2024-11-24T22:16:16,462 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 744bbec6934be304e01b009be760323d:A, priority=-2147483648, current under compaction store size is 1 2024-11-24T22:16:16,462 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-24T22:16:16,462 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 744bbec6934be304e01b009be760323d:B, priority=-2147483648, current under compaction store size is 2 2024-11-24T22:16:16,462 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-24T22:16:16,462 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 744bbec6934be304e01b009be760323d:C, priority=-2147483648, current under compaction store size is 3 2024-11-24T22:16:16,462 DEBUG [RS:0;bd6592c37bb2:40087-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-24T22:16:16,462 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-24T22:16:16,462 DEBUG [RS:0;bd6592c37bb2:40087-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-24T22:16:16,463 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40087 {}] regionserver.HRegion(8855): Flush requested on 744bbec6934be304e01b009be760323d 2024-11-24T22:16:16,464 DEBUG [RS:0;bd6592c37bb2:40087-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 52462 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-24T22:16:16,464 DEBUG [RS:0;bd6592c37bb2:40087-shortCompactions-0 {}] regionserver.HStore(1541): 744bbec6934be304e01b009be760323d/A is initiating minor compaction (all files) 2024-11-24T22:16:16,464 INFO [RS:0;bd6592c37bb2:40087-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 744bbec6934be304e01b009be760323d/A in TestAcidGuarantees,,1732486556329.744bbec6934be304e01b009be760323d. 2024-11-24T22:16:16,464 INFO [RS:0;bd6592c37bb2:40087-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/744bbec6934be304e01b009be760323d/A/4cbb2b50c7d74794986c166190e79fdb, hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/744bbec6934be304e01b009be760323d/A/8007d32c8dc5404ea79a5762cb35d83d, hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/744bbec6934be304e01b009be760323d/A/ed730b6e50aa4416aa0742d68965b490, hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/744bbec6934be304e01b009be760323d/A/513165e189654d61a14d6ba01582bce3] into tmpdir=hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/744bbec6934be304e01b009be760323d/.tmp, totalSize=51.2 K 2024-11-24T22:16:16,464 DEBUG [RS:0;bd6592c37bb2:40087-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 50022 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-24T22:16:16,464 DEBUG [RS:0;bd6592c37bb2:40087-longCompactions-0 {}] regionserver.HStore(1541): 744bbec6934be304e01b009be760323d/B is initiating minor compaction (all files) 2024-11-24T22:16:16,464 INFO [RS:0;bd6592c37bb2:40087-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 744bbec6934be304e01b009be760323d/B in TestAcidGuarantees,,1732486556329.744bbec6934be304e01b009be760323d. 2024-11-24T22:16:16,464 INFO [RS:0;bd6592c37bb2:40087-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/744bbec6934be304e01b009be760323d/B/6ee3d0de54754f999dfa2ddfe20f7a94, hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/744bbec6934be304e01b009be760323d/B/6fe913fa79a24f6fae7b349083870039, hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/744bbec6934be304e01b009be760323d/B/a25b2da6745546899c020b13033d9224, hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/744bbec6934be304e01b009be760323d/B/09209f77165d4e7f9f6c4c1ed40f9136] into tmpdir=hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/744bbec6934be304e01b009be760323d/.tmp, totalSize=48.8 K 2024-11-24T22:16:16,464 DEBUG [RS:0;bd6592c37bb2:40087-shortCompactions-0 {}] compactions.Compactor(225): Compacting 4cbb2b50c7d74794986c166190e79fdb, keycount=150, bloomtype=ROW, size=12.8 K, encoding=NONE, compression=NONE, seqNum=327, earliestPutTs=1732486573413 2024-11-24T22:16:16,465 DEBUG [RS:0;bd6592c37bb2:40087-longCompactions-0 {}] compactions.Compactor(225): Compacting 6ee3d0de54754f999dfa2ddfe20f7a94, keycount=150, bloomtype=ROW, size=12.8 K, encoding=NONE, compression=NONE, seqNum=327, earliestPutTs=1732486573413 2024-11-24T22:16:16,465 DEBUG [RS:0;bd6592c37bb2:40087-shortCompactions-0 {}] compactions.Compactor(225): Compacting 8007d32c8dc5404ea79a5762cb35d83d, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=344, earliestPutTs=1732486574653 2024-11-24T22:16:16,465 DEBUG [RS:0;bd6592c37bb2:40087-longCompactions-0 {}] compactions.Compactor(225): Compacting 6fe913fa79a24f6fae7b349083870039, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=344, earliestPutTs=1732486574653 2024-11-24T22:16:16,465 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 744bbec6934be304e01b009be760323d 3/3 column families, dataSize=73.80 KB heapSize=219.89 KB 2024-11-24T22:16:16,465 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 744bbec6934be304e01b009be760323d, store=A 2024-11-24T22:16:16,465 DEBUG [RS:0;bd6592c37bb2:40087-shortCompactions-0 {}] compactions.Compactor(225): Compacting ed730b6e50aa4416aa0742d68965b490, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=362, earliestPutTs=1732486574760 2024-11-24T22:16:16,465 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-24T22:16:16,465 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 744bbec6934be304e01b009be760323d, store=B 2024-11-24T22:16:16,465 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-24T22:16:16,465 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 744bbec6934be304e01b009be760323d, store=C 2024-11-24T22:16:16,465 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-24T22:16:16,465 DEBUG [RS:0;bd6592c37bb2:40087-longCompactions-0 {}] compactions.Compactor(225): Compacting a25b2da6745546899c020b13033d9224, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=362, earliestPutTs=1732486574760 2024-11-24T22:16:16,465 DEBUG [RS:0;bd6592c37bb2:40087-shortCompactions-0 {}] compactions.Compactor(225): Compacting 513165e189654d61a14d6ba01582bce3, keycount=200, bloomtype=ROW, size=14.4 K, encoding=NONE, compression=NONE, seqNum=380, earliestPutTs=1732486576048 2024-11-24T22:16:16,466 DEBUG [RS:0;bd6592c37bb2:40087-longCompactions-0 {}] compactions.Compactor(225): Compacting 09209f77165d4e7f9f6c4c1ed40f9136, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=380, earliestPutTs=1732486576048 2024-11-24T22:16:16,474 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/744bbec6934be304e01b009be760323d/.tmp/A/bae78c8dcb824903b1bac1b8970098b3 is 50, key is test_row_0/A:col10/1732486576462/Put/seqid=0 2024-11-24T22:16:16,483 INFO [RS:0;bd6592c37bb2:40087-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 744bbec6934be304e01b009be760323d#B#compaction#459 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 57.69 MB/second 2024-11-24T22:16:16,483 DEBUG [RS:0;bd6592c37bb2:40087-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/744bbec6934be304e01b009be760323d/.tmp/B/015f0d78f00e465198f4def8f41b689f is 50, key is test_row_0/B:col10/1732486576053/Put/seqid=0 2024-11-24T22:16:16,493 INFO [RS:0;bd6592c37bb2:40087-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 744bbec6934be304e01b009be760323d#A#compaction#460 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 57.69 MB/second 2024-11-24T22:16:16,494 DEBUG [RS:0;bd6592c37bb2:40087-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/744bbec6934be304e01b009be760323d/.tmp/A/8939132dcea842a3bfda12ce12488284 is 50, key is test_row_0/A:col10/1732486576053/Put/seqid=0 2024-11-24T22:16:16,495 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45929 is added to blk_1073742362_1538 (size=19621) 2024-11-24T22:16:16,498 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45929 is added to blk_1073742363_1539 (size=13255) 2024-11-24T22:16:16,499 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45929 is added to blk_1073742364_1540 (size=13255) 2024-11-24T22:16:16,503 DEBUG [RS:0;bd6592c37bb2:40087-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/744bbec6934be304e01b009be760323d/.tmp/A/8939132dcea842a3bfda12ce12488284 as hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/744bbec6934be304e01b009be760323d/A/8939132dcea842a3bfda12ce12488284 2024-11-24T22:16:16,507 INFO [RS:0;bd6592c37bb2:40087-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 744bbec6934be304e01b009be760323d/A of 744bbec6934be304e01b009be760323d into 8939132dcea842a3bfda12ce12488284(size=12.9 K), total size for store is 12.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-24T22:16:16,507 DEBUG [RS:0;bd6592c37bb2:40087-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 744bbec6934be304e01b009be760323d: 2024-11-24T22:16:16,507 INFO [RS:0;bd6592c37bb2:40087-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732486556329.744bbec6934be304e01b009be760323d., storeName=744bbec6934be304e01b009be760323d/A, priority=12, startTime=1732486576462; duration=0sec 2024-11-24T22:16:16,507 DEBUG [RS:0;bd6592c37bb2:40087-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-24T22:16:16,507 DEBUG [RS:0;bd6592c37bb2:40087-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 744bbec6934be304e01b009be760323d:A 2024-11-24T22:16:16,507 DEBUG [RS:0;bd6592c37bb2:40087-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-24T22:16:16,508 DEBUG [RS:0;bd6592c37bb2:40087-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 50022 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-24T22:16:16,508 DEBUG [RS:0;bd6592c37bb2:40087-shortCompactions-0 {}] regionserver.HStore(1541): 744bbec6934be304e01b009be760323d/C is initiating minor compaction (all files) 2024-11-24T22:16:16,508 INFO [RS:0;bd6592c37bb2:40087-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 744bbec6934be304e01b009be760323d/C in TestAcidGuarantees,,1732486556329.744bbec6934be304e01b009be760323d. 2024-11-24T22:16:16,508 INFO [RS:0;bd6592c37bb2:40087-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/744bbec6934be304e01b009be760323d/C/18cfa9a432cd4a559609140d0121d730, hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/744bbec6934be304e01b009be760323d/C/5d83f39257e349a0b43d8f3dd1fdf19b, hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/744bbec6934be304e01b009be760323d/C/a23bbfecd00441eab0a749d4ed2a5cd5, hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/744bbec6934be304e01b009be760323d/C/d2585fa8933643ddb5981789762dddba] into tmpdir=hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/744bbec6934be304e01b009be760323d/.tmp, totalSize=48.8 K 2024-11-24T22:16:16,508 DEBUG [RS:0;bd6592c37bb2:40087-shortCompactions-0 {}] compactions.Compactor(225): Compacting 18cfa9a432cd4a559609140d0121d730, keycount=150, bloomtype=ROW, size=12.8 K, encoding=NONE, compression=NONE, seqNum=327, earliestPutTs=1732486573413 2024-11-24T22:16:16,509 DEBUG [RS:0;bd6592c37bb2:40087-shortCompactions-0 {}] compactions.Compactor(225): Compacting 5d83f39257e349a0b43d8f3dd1fdf19b, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=344, earliestPutTs=1732486574653 2024-11-24T22:16:16,509 DEBUG [RS:0;bd6592c37bb2:40087-shortCompactions-0 {}] compactions.Compactor(225): Compacting a23bbfecd00441eab0a749d4ed2a5cd5, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=362, earliestPutTs=1732486574760 2024-11-24T22:16:16,509 DEBUG [RS:0;bd6592c37bb2:40087-shortCompactions-0 {}] compactions.Compactor(225): Compacting d2585fa8933643ddb5981789762dddba, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=380, earliestPutTs=1732486576048 2024-11-24T22:16:16,516 INFO [RS:0;bd6592c37bb2:40087-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 744bbec6934be304e01b009be760323d#C#compaction#461 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 57.69 MB/second 2024-11-24T22:16:16,517 DEBUG [RS:0;bd6592c37bb2:40087-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/744bbec6934be304e01b009be760323d/.tmp/C/05ec45638c4643b1b3313cdf31e2f636 is 50, key is test_row_0/C:col10/1732486576053/Put/seqid=0 2024-11-24T22:16:16,520 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45929 is added to blk_1073742365_1541 (size=13255) 2024-11-24T22:16:16,523 DEBUG [RS:0;bd6592c37bb2:40087-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/744bbec6934be304e01b009be760323d/.tmp/C/05ec45638c4643b1b3313cdf31e2f636 as hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/744bbec6934be304e01b009be760323d/C/05ec45638c4643b1b3313cdf31e2f636 2024-11-24T22:16:16,526 INFO [RS:0;bd6592c37bb2:40087-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 744bbec6934be304e01b009be760323d/C of 744bbec6934be304e01b009be760323d into 05ec45638c4643b1b3313cdf31e2f636(size=12.9 K), total size for store is 12.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-24T22:16:16,526 DEBUG [RS:0;bd6592c37bb2:40087-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 744bbec6934be304e01b009be760323d: 2024-11-24T22:16:16,526 INFO [RS:0;bd6592c37bb2:40087-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732486556329.744bbec6934be304e01b009be760323d., storeName=744bbec6934be304e01b009be760323d/C, priority=12, startTime=1732486576462; duration=0sec 2024-11-24T22:16:16,526 DEBUG [RS:0;bd6592c37bb2:40087-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-24T22:16:16,526 DEBUG [RS:0;bd6592c37bb2:40087-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 744bbec6934be304e01b009be760323d:C 2024-11-24T22:16:16,560 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40087 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=744bbec6934be304e01b009be760323d, server=bd6592c37bb2,40087,1732486423713 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-24T22:16:16,560 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40087 {}] ipc.CallRunner(138): callId: 117 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46842 deadline: 1732486636554, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=744bbec6934be304e01b009be760323d, server=bd6592c37bb2,40087,1732486423713 2024-11-24T22:16:16,561 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732486556329.744bbec6934be304e01b009be760323d., hostname=bd6592c37bb2,40087,1732486423713, seqNum=2 , the old value is region=TestAcidGuarantees,,1732486556329.744bbec6934be304e01b009be760323d., hostname=bd6592c37bb2,40087,1732486423713, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=744bbec6934be304e01b009be760323d, server=bd6592c37bb2,40087,1732486423713 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T22:16:16,561 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732486556329.744bbec6934be304e01b009be760323d., hostname=bd6592c37bb2,40087,1732486423713, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=744bbec6934be304e01b009be760323d, server=bd6592c37bb2,40087,1732486423713 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T22:16:16,561 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732486556329.744bbec6934be304e01b009be760323d., hostname=bd6592c37bb2,40087,1732486423713, seqNum=2 because the exception is null or not the one we care about 2024-11-24T22:16:16,567 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40087 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=744bbec6934be304e01b009be760323d, server=bd6592c37bb2,40087,1732486423713 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-24T22:16:16,567 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40087 {}] ipc.CallRunner(138): callId: 190 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46792 deadline: 1732486636555, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=744bbec6934be304e01b009be760323d, server=bd6592c37bb2,40087,1732486423713 2024-11-24T22:16:16,567 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732486556329.744bbec6934be304e01b009be760323d., hostname=bd6592c37bb2,40087,1732486423713, seqNum=2 , the old value is region=TestAcidGuarantees,,1732486556329.744bbec6934be304e01b009be760323d., hostname=bd6592c37bb2,40087,1732486423713, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=744bbec6934be304e01b009be760323d, server=bd6592c37bb2,40087,1732486423713 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T22:16:16,568 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732486556329.744bbec6934be304e01b009be760323d., hostname=bd6592c37bb2,40087,1732486423713, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=744bbec6934be304e01b009be760323d, server=bd6592c37bb2,40087,1732486423713 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T22:16:16,568 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732486556329.744bbec6934be304e01b009be760323d., hostname=bd6592c37bb2,40087,1732486423713, seqNum=2 because the exception is null or not the one we care about 2024-11-24T22:16:16,675 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40087 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=744bbec6934be304e01b009be760323d, server=bd6592c37bb2,40087,1732486423713 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-24T22:16:16,675 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40087 {}] ipc.CallRunner(138): callId: 118 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46842 deadline: 1732486636670, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=744bbec6934be304e01b009be760323d, server=bd6592c37bb2,40087,1732486423713 2024-11-24T22:16:16,675 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40087 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=744bbec6934be304e01b009be760323d, server=bd6592c37bb2,40087,1732486423713 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-24T22:16:16,676 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732486556329.744bbec6934be304e01b009be760323d., hostname=bd6592c37bb2,40087,1732486423713, seqNum=2 , the old value is region=TestAcidGuarantees,,1732486556329.744bbec6934be304e01b009be760323d., hostname=bd6592c37bb2,40087,1732486423713, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=744bbec6934be304e01b009be760323d, server=bd6592c37bb2,40087,1732486423713 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T22:16:16,676 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732486556329.744bbec6934be304e01b009be760323d., hostname=bd6592c37bb2,40087,1732486423713, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=744bbec6934be304e01b009be760323d, server=bd6592c37bb2,40087,1732486423713 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T22:16:16,676 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40087 {}] ipc.CallRunner(138): callId: 191 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46792 deadline: 1732486636671, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=744bbec6934be304e01b009be760323d, server=bd6592c37bb2,40087,1732486423713 2024-11-24T22:16:16,676 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732486556329.744bbec6934be304e01b009be760323d., hostname=bd6592c37bb2,40087,1732486423713, seqNum=2 because the exception is null or not the one we care about 2024-11-24T22:16:16,676 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732486556329.744bbec6934be304e01b009be760323d., hostname=bd6592c37bb2,40087,1732486423713, seqNum=2 , the old value is region=TestAcidGuarantees,,1732486556329.744bbec6934be304e01b009be760323d., hostname=bd6592c37bb2,40087,1732486423713, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=744bbec6934be304e01b009be760323d, server=bd6592c37bb2,40087,1732486423713 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T22:16:16,676 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732486556329.744bbec6934be304e01b009be760323d., hostname=bd6592c37bb2,40087,1732486423713, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=744bbec6934be304e01b009be760323d, server=bd6592c37bb2,40087,1732486423713 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T22:16:16,676 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732486556329.744bbec6934be304e01b009be760323d., hostname=bd6592c37bb2,40087,1732486423713, seqNum=2 because the exception is null or not the one we care about 2024-11-24T22:16:16,891 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40087 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=744bbec6934be304e01b009be760323d, server=bd6592c37bb2,40087,1732486423713 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-24T22:16:16,891 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40087 {}] ipc.CallRunner(138): callId: 119 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46842 deadline: 1732486636881, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=744bbec6934be304e01b009be760323d, server=bd6592c37bb2,40087,1732486423713 2024-11-24T22:16:16,891 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40087 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=744bbec6934be304e01b009be760323d, server=bd6592c37bb2,40087,1732486423713 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-24T22:16:16,891 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732486556329.744bbec6934be304e01b009be760323d., hostname=bd6592c37bb2,40087,1732486423713, seqNum=2 , the old value is region=TestAcidGuarantees,,1732486556329.744bbec6934be304e01b009be760323d., hostname=bd6592c37bb2,40087,1732486423713, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=744bbec6934be304e01b009be760323d, server=bd6592c37bb2,40087,1732486423713 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T22:16:16,891 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732486556329.744bbec6934be304e01b009be760323d., hostname=bd6592c37bb2,40087,1732486423713, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=744bbec6934be304e01b009be760323d, server=bd6592c37bb2,40087,1732486423713 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T22:16:16,891 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40087 {}] ipc.CallRunner(138): callId: 192 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46792 deadline: 1732486636881, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=744bbec6934be304e01b009be760323d, server=bd6592c37bb2,40087,1732486423713 2024-11-24T22:16:16,891 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732486556329.744bbec6934be304e01b009be760323d., hostname=bd6592c37bb2,40087,1732486423713, seqNum=2 because the exception is null or not the one we care about 2024-11-24T22:16:16,892 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732486556329.744bbec6934be304e01b009be760323d., hostname=bd6592c37bb2,40087,1732486423713, seqNum=2 , the old value is region=TestAcidGuarantees,,1732486556329.744bbec6934be304e01b009be760323d., hostname=bd6592c37bb2,40087,1732486423713, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=744bbec6934be304e01b009be760323d, server=bd6592c37bb2,40087,1732486423713 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T22:16:16,892 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732486556329.744bbec6934be304e01b009be760323d., hostname=bd6592c37bb2,40087,1732486423713, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=744bbec6934be304e01b009be760323d, server=bd6592c37bb2,40087,1732486423713 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T22:16:16,892 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732486556329.744bbec6934be304e01b009be760323d., hostname=bd6592c37bb2,40087,1732486423713, seqNum=2 because the exception is null or not the one we care about 2024-11-24T22:16:16,895 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=24.60 KB at sequenceid=394 (bloomFilter=true), to=hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/744bbec6934be304e01b009be760323d/.tmp/A/bae78c8dcb824903b1bac1b8970098b3 2024-11-24T22:16:16,902 DEBUG [RS:0;bd6592c37bb2:40087-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/744bbec6934be304e01b009be760323d/.tmp/B/015f0d78f00e465198f4def8f41b689f as hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/744bbec6934be304e01b009be760323d/B/015f0d78f00e465198f4def8f41b689f 2024-11-24T22:16:16,902 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/744bbec6934be304e01b009be760323d/.tmp/B/442e65fffb94469f8109f1c971c49198 is 50, key is test_row_0/B:col10/1732486576462/Put/seqid=0 2024-11-24T22:16:16,905 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45929 is added to blk_1073742366_1542 (size=12301) 2024-11-24T22:16:16,905 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=24.60 KB at sequenceid=394 (bloomFilter=true), to=hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/744bbec6934be304e01b009be760323d/.tmp/B/442e65fffb94469f8109f1c971c49198 2024-11-24T22:16:16,906 INFO [RS:0;bd6592c37bb2:40087-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 744bbec6934be304e01b009be760323d/B of 744bbec6934be304e01b009be760323d into 015f0d78f00e465198f4def8f41b689f(size=12.9 K), total size for store is 12.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-24T22:16:16,906 DEBUG [RS:0;bd6592c37bb2:40087-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 744bbec6934be304e01b009be760323d: 2024-11-24T22:16:16,906 INFO [RS:0;bd6592c37bb2:40087-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732486556329.744bbec6934be304e01b009be760323d., storeName=744bbec6934be304e01b009be760323d/B, priority=12, startTime=1732486576462; duration=0sec 2024-11-24T22:16:16,906 DEBUG [RS:0;bd6592c37bb2:40087-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-24T22:16:16,906 DEBUG [RS:0;bd6592c37bb2:40087-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 744bbec6934be304e01b009be760323d:B 2024-11-24T22:16:16,911 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37701 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=135 2024-11-24T22:16:16,911 INFO [RPCClient-NioEventLoopGroup-4-10 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-24T22:16:16,911 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37701 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-24T22:16:16,912 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37701 {}] procedure2.ProcedureExecutor(1139): Stored pid=137, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=137, table=TestAcidGuarantees 2024-11-24T22:16:16,913 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=137, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=137, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-24T22:16:16,913 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37701 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=137 2024-11-24T22:16:16,914 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=137, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=137, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-24T22:16:16,914 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=138, ppid=137, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-24T22:16:16,915 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/744bbec6934be304e01b009be760323d/.tmp/C/f67ba0237d4c48d8b9ce5f846119b2ba is 50, key is test_row_0/C:col10/1732486576462/Put/seqid=0 2024-11-24T22:16:16,918 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45929 is added to blk_1073742367_1543 (size=12301) 2024-11-24T22:16:17,006 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40087 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=744bbec6934be304e01b009be760323d, server=bd6592c37bb2,40087,1732486423713 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-24T22:16:17,006 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40087 {}] ipc.CallRunner(138): callId: 70 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46820 deadline: 1732486637000, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=744bbec6934be304e01b009be760323d, server=bd6592c37bb2,40087,1732486423713 2024-11-24T22:16:17,006 WARN [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRpcRetryingCaller(177): Call to bd6592c37bb2,40087,1732486423713 for 'test_row_0' in 744bbec6934be304e01b009be760323d of TestAcidGuarantees failed, tries = 7, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 4255 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=744bbec6934be304e01b009be760323d, server=bd6592c37bb2,40087,1732486423713 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor44.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=744bbec6934be304e01b009be760323d, server=bd6592c37bb2,40087,1732486423713 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-24T22:16:17,007 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732486556329.744bbec6934be304e01b009be760323d., hostname=bd6592c37bb2,40087,1732486423713, seqNum=2 , the old value is region=TestAcidGuarantees,,1732486556329.744bbec6934be304e01b009be760323d., hostname=bd6592c37bb2,40087,1732486423713, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=744bbec6934be304e01b009be760323d, server=bd6592c37bb2,40087,1732486423713 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T22:16:17,007 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732486556329.744bbec6934be304e01b009be760323d., hostname=bd6592c37bb2,40087,1732486423713, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=744bbec6934be304e01b009be760323d, server=bd6592c37bb2,40087,1732486423713 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T22:16:17,007 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732486556329.744bbec6934be304e01b009be760323d., hostname=bd6592c37bb2,40087,1732486423713, seqNum=2 because the exception is null or not the one we care about 2024-11-24T22:16:17,010 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40087 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=744bbec6934be304e01b009be760323d, server=bd6592c37bb2,40087,1732486423713 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-24T22:16:17,010 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40087 {}] ipc.CallRunner(138): callId: 78 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46824 deadline: 1732486637000, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=744bbec6934be304e01b009be760323d, server=bd6592c37bb2,40087,1732486423713 2024-11-24T22:16:17,010 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40087 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=744bbec6934be304e01b009be760323d, server=bd6592c37bb2,40087,1732486423713 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-24T22:16:17,010 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40087 {}] ipc.CallRunner(138): callId: 69 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46816 deadline: 1732486637001, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=744bbec6934be304e01b009be760323d, server=bd6592c37bb2,40087,1732486423713 2024-11-24T22:16:17,010 WARN [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRpcRetryingCaller(177): Call to bd6592c37bb2,40087,1732486423713 for 'test_row_2' in 744bbec6934be304e01b009be760323d of TestAcidGuarantees failed, tries = 7, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 4256 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=744bbec6934be304e01b009be760323d, server=bd6592c37bb2,40087,1732486423713 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor44.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=744bbec6934be304e01b009be760323d, server=bd6592c37bb2,40087,1732486423713 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-24T22:16:17,011 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732486556329.744bbec6934be304e01b009be760323d., hostname=bd6592c37bb2,40087,1732486423713, seqNum=2 , the old value is region=TestAcidGuarantees,,1732486556329.744bbec6934be304e01b009be760323d., hostname=bd6592c37bb2,40087,1732486423713, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=744bbec6934be304e01b009be760323d, server=bd6592c37bb2,40087,1732486423713 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T22:16:17,011 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732486556329.744bbec6934be304e01b009be760323d., hostname=bd6592c37bb2,40087,1732486423713, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=744bbec6934be304e01b009be760323d, server=bd6592c37bb2,40087,1732486423713 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T22:16:17,011 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732486556329.744bbec6934be304e01b009be760323d., hostname=bd6592c37bb2,40087,1732486423713, seqNum=2 because the exception is null or not the one we care about 2024-11-24T22:16:17,011 WARN [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRpcRetryingCaller(177): Call to bd6592c37bb2,40087,1732486423713 for 'test_row_2' in 744bbec6934be304e01b009be760323d of TestAcidGuarantees failed, tries = 7, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 4256 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=744bbec6934be304e01b009be760323d, server=bd6592c37bb2,40087,1732486423713 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor44.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=744bbec6934be304e01b009be760323d, server=bd6592c37bb2,40087,1732486423713 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-24T22:16:17,011 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732486556329.744bbec6934be304e01b009be760323d., hostname=bd6592c37bb2,40087,1732486423713, seqNum=2 , the old value is region=TestAcidGuarantees,,1732486556329.744bbec6934be304e01b009be760323d., hostname=bd6592c37bb2,40087,1732486423713, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=744bbec6934be304e01b009be760323d, server=bd6592c37bb2,40087,1732486423713 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T22:16:17,011 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732486556329.744bbec6934be304e01b009be760323d., hostname=bd6592c37bb2,40087,1732486423713, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=744bbec6934be304e01b009be760323d, server=bd6592c37bb2,40087,1732486423713 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T22:16:17,011 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732486556329.744bbec6934be304e01b009be760323d., hostname=bd6592c37bb2,40087,1732486423713, seqNum=2 because the exception is null or not the one we care about 2024-11-24T22:16:17,021 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37701 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=137 2024-11-24T22:16:17,066 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=40087 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=138 2024-11-24T22:16:17,066 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bd6592c37bb2:0-0 {event_type=RS_FLUSH_REGIONS, pid=138}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732486556329.744bbec6934be304e01b009be760323d. 2024-11-24T22:16:17,066 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bd6592c37bb2:0-0 {event_type=RS_FLUSH_REGIONS, pid=138}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732486556329.744bbec6934be304e01b009be760323d. as already flushing 2024-11-24T22:16:17,066 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bd6592c37bb2:0-0 {event_type=RS_FLUSH_REGIONS, pid=138}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732486556329.744bbec6934be304e01b009be760323d. 2024-11-24T22:16:17,066 ERROR [RS_FLUSH_OPERATIONS-regionserver/bd6592c37bb2:0-0 {event_type=RS_FLUSH_REGIONS, pid=138}] handler.RSProcedureHandler(58): pid=138 java.io.IOException: Unable to complete flush {ENCODED => 744bbec6934be304e01b009be760323d, NAME => 'TestAcidGuarantees,,1732486556329.744bbec6934be304e01b009be760323d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T22:16:17,066 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bd6592c37bb2:0-0 {event_type=RS_FLUSH_REGIONS, pid=138}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=138 java.io.IOException: Unable to complete flush {ENCODED => 744bbec6934be304e01b009be760323d, NAME => 'TestAcidGuarantees,,1732486556329.744bbec6934be304e01b009be760323d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T22:16:17,067 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37701 {}] master.HMaster(4177): Remote procedure failed, pid=138 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 744bbec6934be304e01b009be760323d, NAME => 'TestAcidGuarantees,,1732486556329.744bbec6934be304e01b009be760323d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 744bbec6934be304e01b009be760323d, NAME => 'TestAcidGuarantees,,1732486556329.744bbec6934be304e01b009be760323d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T22:16:17,210 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40087 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=744bbec6934be304e01b009be760323d, server=bd6592c37bb2,40087,1732486423713 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-24T22:16:17,210 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40087 {}] ipc.CallRunner(138): callId: 120 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46842 deadline: 1732486637200, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=744bbec6934be304e01b009be760323d, server=bd6592c37bb2,40087,1732486423713 2024-11-24T22:16:17,210 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40087 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=744bbec6934be304e01b009be760323d, server=bd6592c37bb2,40087,1732486423713 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-24T22:16:17,210 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40087 {}] ipc.CallRunner(138): callId: 193 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46792 deadline: 1732486637200, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=744bbec6934be304e01b009be760323d, server=bd6592c37bb2,40087,1732486423713 2024-11-24T22:16:17,210 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732486556329.744bbec6934be304e01b009be760323d., hostname=bd6592c37bb2,40087,1732486423713, seqNum=2 , the old value is region=TestAcidGuarantees,,1732486556329.744bbec6934be304e01b009be760323d., hostname=bd6592c37bb2,40087,1732486423713, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=744bbec6934be304e01b009be760323d, server=bd6592c37bb2,40087,1732486423713 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T22:16:17,210 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732486556329.744bbec6934be304e01b009be760323d., hostname=bd6592c37bb2,40087,1732486423713, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=744bbec6934be304e01b009be760323d, server=bd6592c37bb2,40087,1732486423713 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T22:16:17,210 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732486556329.744bbec6934be304e01b009be760323d., hostname=bd6592c37bb2,40087,1732486423713, seqNum=2 because the exception is null or not the one we care about 2024-11-24T22:16:17,211 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732486556329.744bbec6934be304e01b009be760323d., hostname=bd6592c37bb2,40087,1732486423713, seqNum=2 , the old value is region=TestAcidGuarantees,,1732486556329.744bbec6934be304e01b009be760323d., hostname=bd6592c37bb2,40087,1732486423713, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=744bbec6934be304e01b009be760323d, server=bd6592c37bb2,40087,1732486423713 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T22:16:17,211 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732486556329.744bbec6934be304e01b009be760323d., hostname=bd6592c37bb2,40087,1732486423713, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=744bbec6934be304e01b009be760323d, server=bd6592c37bb2,40087,1732486423713 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T22:16:17,211 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732486556329.744bbec6934be304e01b009be760323d., hostname=bd6592c37bb2,40087,1732486423713, seqNum=2 because the exception is null or not the one we care about 2024-11-24T22:16:17,218 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=40087 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=138 2024-11-24T22:16:17,218 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bd6592c37bb2:0-1 {event_type=RS_FLUSH_REGIONS, pid=138}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732486556329.744bbec6934be304e01b009be760323d. 2024-11-24T22:16:17,219 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bd6592c37bb2:0-1 {event_type=RS_FLUSH_REGIONS, pid=138}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732486556329.744bbec6934be304e01b009be760323d. as already flushing 2024-11-24T22:16:17,219 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bd6592c37bb2:0-1 {event_type=RS_FLUSH_REGIONS, pid=138}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732486556329.744bbec6934be304e01b009be760323d. 2024-11-24T22:16:17,219 ERROR [RS_FLUSH_OPERATIONS-regionserver/bd6592c37bb2:0-1 {event_type=RS_FLUSH_REGIONS, pid=138}] handler.RSProcedureHandler(58): pid=138 java.io.IOException: Unable to complete flush {ENCODED => 744bbec6934be304e01b009be760323d, NAME => 'TestAcidGuarantees,,1732486556329.744bbec6934be304e01b009be760323d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T22:16:17,219 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bd6592c37bb2:0-1 {event_type=RS_FLUSH_REGIONS, pid=138}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=138 java.io.IOException: Unable to complete flush {ENCODED => 744bbec6934be304e01b009be760323d, NAME => 'TestAcidGuarantees,,1732486556329.744bbec6934be304e01b009be760323d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T22:16:17,219 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37701 {}] master.HMaster(4177): Remote procedure failed, pid=138 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 744bbec6934be304e01b009be760323d, NAME => 'TestAcidGuarantees,,1732486556329.744bbec6934be304e01b009be760323d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 744bbec6934be304e01b009be760323d, NAME => 'TestAcidGuarantees,,1732486556329.744bbec6934be304e01b009be760323d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T22:16:17,231 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37701 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=137 2024-11-24T22:16:17,319 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=24.60 KB at sequenceid=394 (bloomFilter=true), to=hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/744bbec6934be304e01b009be760323d/.tmp/C/f67ba0237d4c48d8b9ce5f846119b2ba 2024-11-24T22:16:17,322 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/744bbec6934be304e01b009be760323d/.tmp/A/bae78c8dcb824903b1bac1b8970098b3 as hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/744bbec6934be304e01b009be760323d/A/bae78c8dcb824903b1bac1b8970098b3 2024-11-24T22:16:17,325 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/744bbec6934be304e01b009be760323d/A/bae78c8dcb824903b1bac1b8970098b3, entries=300, sequenceid=394, filesize=19.2 K 2024-11-24T22:16:17,326 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/744bbec6934be304e01b009be760323d/.tmp/B/442e65fffb94469f8109f1c971c49198 as hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/744bbec6934be304e01b009be760323d/B/442e65fffb94469f8109f1c971c49198 2024-11-24T22:16:17,329 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/744bbec6934be304e01b009be760323d/B/442e65fffb94469f8109f1c971c49198, entries=150, sequenceid=394, filesize=12.0 K 2024-11-24T22:16:17,329 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/744bbec6934be304e01b009be760323d/.tmp/C/f67ba0237d4c48d8b9ce5f846119b2ba as hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/744bbec6934be304e01b009be760323d/C/f67ba0237d4c48d8b9ce5f846119b2ba 2024-11-24T22:16:17,332 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/744bbec6934be304e01b009be760323d/C/f67ba0237d4c48d8b9ce5f846119b2ba, entries=150, sequenceid=394, filesize=12.0 K 2024-11-24T22:16:17,332 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~73.80 KB/75570, heapSize ~219.84 KB/225120, currentSize=100.63 KB/103050 for 744bbec6934be304e01b009be760323d in 867ms, sequenceid=394, compaction requested=false 2024-11-24T22:16:17,332 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 744bbec6934be304e01b009be760323d: 2024-11-24T22:16:17,371 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=40087 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=138 2024-11-24T22:16:17,371 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bd6592c37bb2:0-2 {event_type=RS_FLUSH_REGIONS, pid=138}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732486556329.744bbec6934be304e01b009be760323d. 2024-11-24T22:16:17,371 INFO [RS_FLUSH_OPERATIONS-regionserver/bd6592c37bb2:0-2 {event_type=RS_FLUSH_REGIONS, pid=138}] regionserver.HRegion(2902): Flushing 744bbec6934be304e01b009be760323d 3/3 column families, dataSize=100.63 KB heapSize=299.58 KB 2024-11-24T22:16:17,371 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bd6592c37bb2:0-2 {event_type=RS_FLUSH_REGIONS, pid=138}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 744bbec6934be304e01b009be760323d, store=A 2024-11-24T22:16:17,371 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bd6592c37bb2:0-2 {event_type=RS_FLUSH_REGIONS, pid=138}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-24T22:16:17,371 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bd6592c37bb2:0-2 {event_type=RS_FLUSH_REGIONS, pid=138}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 744bbec6934be304e01b009be760323d, store=B 2024-11-24T22:16:17,371 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bd6592c37bb2:0-2 {event_type=RS_FLUSH_REGIONS, pid=138}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-24T22:16:17,372 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bd6592c37bb2:0-2 {event_type=RS_FLUSH_REGIONS, pid=138}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 744bbec6934be304e01b009be760323d, store=C 2024-11-24T22:16:17,372 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bd6592c37bb2:0-2 {event_type=RS_FLUSH_REGIONS, pid=138}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-24T22:16:17,375 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bd6592c37bb2:0-2 {event_type=RS_FLUSH_REGIONS, pid=138}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/744bbec6934be304e01b009be760323d/.tmp/A/ce29d063c8214791bbc3b99864b9d49e is 50, key is test_row_0/A:col10/1732486576553/Put/seqid=0 2024-11-24T22:16:17,378 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45929 is added to blk_1073742368_1544 (size=12301) 2024-11-24T22:16:17,378 INFO [RS_FLUSH_OPERATIONS-regionserver/bd6592c37bb2:0-2 {event_type=RS_FLUSH_REGIONS, pid=138}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=33.54 KB at sequenceid=415 (bloomFilter=true), to=hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/744bbec6934be304e01b009be760323d/.tmp/A/ce29d063c8214791bbc3b99864b9d49e 2024-11-24T22:16:17,384 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bd6592c37bb2:0-2 {event_type=RS_FLUSH_REGIONS, pid=138}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/744bbec6934be304e01b009be760323d/.tmp/B/b8f9a37fd50c47c0b695ccbd0b13417b is 50, key is test_row_0/B:col10/1732486576553/Put/seqid=0 2024-11-24T22:16:17,387 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45929 is added to blk_1073742369_1545 (size=12301) 2024-11-24T22:16:17,541 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37701 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=137 2024-11-24T22:16:17,730 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40087 {}] regionserver.HRegion(8855): Flush requested on 744bbec6934be304e01b009be760323d 2024-11-24T22:16:17,730 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732486556329.744bbec6934be304e01b009be760323d. as already flushing 2024-11-24T22:16:17,771 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40087 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=744bbec6934be304e01b009be760323d, server=bd6592c37bb2,40087,1732486423713 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-24T22:16:17,771 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40087 {}] ipc.CallRunner(138): callId: 199 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46792 deadline: 1732486637767, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=744bbec6934be304e01b009be760323d, server=bd6592c37bb2,40087,1732486423713 2024-11-24T22:16:17,771 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732486556329.744bbec6934be304e01b009be760323d., hostname=bd6592c37bb2,40087,1732486423713, seqNum=2 , the old value is region=TestAcidGuarantees,,1732486556329.744bbec6934be304e01b009be760323d., hostname=bd6592c37bb2,40087,1732486423713, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=744bbec6934be304e01b009be760323d, server=bd6592c37bb2,40087,1732486423713 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T22:16:17,771 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732486556329.744bbec6934be304e01b009be760323d., hostname=bd6592c37bb2,40087,1732486423713, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=744bbec6934be304e01b009be760323d, server=bd6592c37bb2,40087,1732486423713 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T22:16:17,772 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732486556329.744bbec6934be304e01b009be760323d., hostname=bd6592c37bb2,40087,1732486423713, seqNum=2 because the exception is null or not the one we care about 2024-11-24T22:16:17,779 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40087 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=744bbec6934be304e01b009be760323d, server=bd6592c37bb2,40087,1732486423713 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-24T22:16:17,780 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40087 {}] ipc.CallRunner(138): callId: 127 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46842 deadline: 1732486637769, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=744bbec6934be304e01b009be760323d, server=bd6592c37bb2,40087,1732486423713 2024-11-24T22:16:17,780 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732486556329.744bbec6934be304e01b009be760323d., hostname=bd6592c37bb2,40087,1732486423713, seqNum=2 , the old value is region=TestAcidGuarantees,,1732486556329.744bbec6934be304e01b009be760323d., hostname=bd6592c37bb2,40087,1732486423713, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=744bbec6934be304e01b009be760323d, server=bd6592c37bb2,40087,1732486423713 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T22:16:17,780 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732486556329.744bbec6934be304e01b009be760323d., hostname=bd6592c37bb2,40087,1732486423713, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=744bbec6934be304e01b009be760323d, server=bd6592c37bb2,40087,1732486423713 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T22:16:17,780 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732486556329.744bbec6934be304e01b009be760323d., hostname=bd6592c37bb2,40087,1732486423713, seqNum=2 because the exception is null or not the one we care about 2024-11-24T22:16:17,788 INFO [RS_FLUSH_OPERATIONS-regionserver/bd6592c37bb2:0-2 {event_type=RS_FLUSH_REGIONS, pid=138}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=33.54 KB at sequenceid=415 (bloomFilter=true), to=hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/744bbec6934be304e01b009be760323d/.tmp/B/b8f9a37fd50c47c0b695ccbd0b13417b 2024-11-24T22:16:17,794 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bd6592c37bb2:0-2 {event_type=RS_FLUSH_REGIONS, pid=138}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/744bbec6934be304e01b009be760323d/.tmp/C/7b7ae2272b0f466688e0574d67613286 is 50, key is test_row_0/C:col10/1732486576553/Put/seqid=0 2024-11-24T22:16:17,798 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45929 is added to blk_1073742370_1546 (size=12301) 2024-11-24T22:16:17,890 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40087 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=744bbec6934be304e01b009be760323d, server=bd6592c37bb2,40087,1732486423713 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-24T22:16:17,890 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40087 {}] ipc.CallRunner(138): callId: 200 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46792 deadline: 1732486637881, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=744bbec6934be304e01b009be760323d, server=bd6592c37bb2,40087,1732486423713 2024-11-24T22:16:17,891 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732486556329.744bbec6934be304e01b009be760323d., hostname=bd6592c37bb2,40087,1732486423713, seqNum=2 , the old value is region=TestAcidGuarantees,,1732486556329.744bbec6934be304e01b009be760323d., hostname=bd6592c37bb2,40087,1732486423713, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=744bbec6934be304e01b009be760323d, server=bd6592c37bb2,40087,1732486423713 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T22:16:17,891 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732486556329.744bbec6934be304e01b009be760323d., hostname=bd6592c37bb2,40087,1732486423713, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=744bbec6934be304e01b009be760323d, server=bd6592c37bb2,40087,1732486423713 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T22:16:17,891 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732486556329.744bbec6934be304e01b009be760323d., hostname=bd6592c37bb2,40087,1732486423713, seqNum=2 because the exception is null or not the one we care about 2024-11-24T22:16:17,900 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40087 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=744bbec6934be304e01b009be760323d, server=bd6592c37bb2,40087,1732486423713 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-24T22:16:17,900 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40087 {}] ipc.CallRunner(138): callId: 128 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46842 deadline: 1732486637891, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=744bbec6934be304e01b009be760323d, server=bd6592c37bb2,40087,1732486423713 2024-11-24T22:16:17,900 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732486556329.744bbec6934be304e01b009be760323d., hostname=bd6592c37bb2,40087,1732486423713, seqNum=2 , the old value is region=TestAcidGuarantees,,1732486556329.744bbec6934be304e01b009be760323d., hostname=bd6592c37bb2,40087,1732486423713, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=744bbec6934be304e01b009be760323d, server=bd6592c37bb2,40087,1732486423713 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T22:16:17,900 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732486556329.744bbec6934be304e01b009be760323d., hostname=bd6592c37bb2,40087,1732486423713, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=744bbec6934be304e01b009be760323d, server=bd6592c37bb2,40087,1732486423713 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T22:16:17,900 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732486556329.744bbec6934be304e01b009be760323d., hostname=bd6592c37bb2,40087,1732486423713, seqNum=2 because the exception is null or not the one we care about 2024-11-24T22:16:18,051 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37701 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=137 2024-11-24T22:16:18,110 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40087 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=744bbec6934be304e01b009be760323d, server=bd6592c37bb2,40087,1732486423713 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-24T22:16:18,110 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40087 {}] ipc.CallRunner(138): callId: 201 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46792 deadline: 1732486638101, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=744bbec6934be304e01b009be760323d, server=bd6592c37bb2,40087,1732486423713 2024-11-24T22:16:18,110 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732486556329.744bbec6934be304e01b009be760323d., hostname=bd6592c37bb2,40087,1732486423713, seqNum=2 , the old value is region=TestAcidGuarantees,,1732486556329.744bbec6934be304e01b009be760323d., hostname=bd6592c37bb2,40087,1732486423713, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=744bbec6934be304e01b009be760323d, server=bd6592c37bb2,40087,1732486423713 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T22:16:18,110 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732486556329.744bbec6934be304e01b009be760323d., hostname=bd6592c37bb2,40087,1732486423713, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=744bbec6934be304e01b009be760323d, server=bd6592c37bb2,40087,1732486423713 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T22:16:18,110 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732486556329.744bbec6934be304e01b009be760323d., hostname=bd6592c37bb2,40087,1732486423713, seqNum=2 because the exception is null or not the one we care about 2024-11-24T22:16:18,116 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40087 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=744bbec6934be304e01b009be760323d, server=bd6592c37bb2,40087,1732486423713 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-24T22:16:18,116 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40087 {}] ipc.CallRunner(138): callId: 129 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46842 deadline: 1732486638111, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=744bbec6934be304e01b009be760323d, server=bd6592c37bb2,40087,1732486423713 2024-11-24T22:16:18,116 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732486556329.744bbec6934be304e01b009be760323d., hostname=bd6592c37bb2,40087,1732486423713, seqNum=2 , the old value is region=TestAcidGuarantees,,1732486556329.744bbec6934be304e01b009be760323d., hostname=bd6592c37bb2,40087,1732486423713, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=744bbec6934be304e01b009be760323d, server=bd6592c37bb2,40087,1732486423713 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T22:16:18,116 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732486556329.744bbec6934be304e01b009be760323d., hostname=bd6592c37bb2,40087,1732486423713, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=744bbec6934be304e01b009be760323d, server=bd6592c37bb2,40087,1732486423713 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T22:16:18,116 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732486556329.744bbec6934be304e01b009be760323d., hostname=bd6592c37bb2,40087,1732486423713, seqNum=2 because the exception is null or not the one we care about 2024-11-24T22:16:18,198 INFO [RS_FLUSH_OPERATIONS-regionserver/bd6592c37bb2:0-2 {event_type=RS_FLUSH_REGIONS, pid=138}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=33.54 KB at sequenceid=415 (bloomFilter=true), to=hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/744bbec6934be304e01b009be760323d/.tmp/C/7b7ae2272b0f466688e0574d67613286 2024-11-24T22:16:18,202 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bd6592c37bb2:0-2 {event_type=RS_FLUSH_REGIONS, pid=138}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/744bbec6934be304e01b009be760323d/.tmp/A/ce29d063c8214791bbc3b99864b9d49e as hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/744bbec6934be304e01b009be760323d/A/ce29d063c8214791bbc3b99864b9d49e 2024-11-24T22:16:18,205 INFO [RS_FLUSH_OPERATIONS-regionserver/bd6592c37bb2:0-2 {event_type=RS_FLUSH_REGIONS, pid=138}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/744bbec6934be304e01b009be760323d/A/ce29d063c8214791bbc3b99864b9d49e, entries=150, sequenceid=415, filesize=12.0 K 2024-11-24T22:16:18,206 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bd6592c37bb2:0-2 {event_type=RS_FLUSH_REGIONS, pid=138}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/744bbec6934be304e01b009be760323d/.tmp/B/b8f9a37fd50c47c0b695ccbd0b13417b as hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/744bbec6934be304e01b009be760323d/B/b8f9a37fd50c47c0b695ccbd0b13417b 2024-11-24T22:16:18,208 INFO [RS_FLUSH_OPERATIONS-regionserver/bd6592c37bb2:0-2 {event_type=RS_FLUSH_REGIONS, pid=138}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/744bbec6934be304e01b009be760323d/B/b8f9a37fd50c47c0b695ccbd0b13417b, entries=150, sequenceid=415, filesize=12.0 K 2024-11-24T22:16:18,209 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bd6592c37bb2:0-2 {event_type=RS_FLUSH_REGIONS, pid=138}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/744bbec6934be304e01b009be760323d/.tmp/C/7b7ae2272b0f466688e0574d67613286 as hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/744bbec6934be304e01b009be760323d/C/7b7ae2272b0f466688e0574d67613286 2024-11-24T22:16:18,214 INFO [RS_FLUSH_OPERATIONS-regionserver/bd6592c37bb2:0-2 {event_type=RS_FLUSH_REGIONS, pid=138}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/744bbec6934be304e01b009be760323d/C/7b7ae2272b0f466688e0574d67613286, entries=150, sequenceid=415, filesize=12.0 K 2024-11-24T22:16:18,214 INFO [RS_FLUSH_OPERATIONS-regionserver/bd6592c37bb2:0-2 {event_type=RS_FLUSH_REGIONS, pid=138}] regionserver.HRegion(3140): Finished flush of dataSize ~100.63 KB/103050, heapSize ~299.53 KB/306720, currentSize=73.80 KB/75570 for 744bbec6934be304e01b009be760323d in 843ms, sequenceid=415, compaction requested=true 2024-11-24T22:16:18,214 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bd6592c37bb2:0-2 {event_type=RS_FLUSH_REGIONS, pid=138}] regionserver.HRegion(2603): Flush status journal for 744bbec6934be304e01b009be760323d: 2024-11-24T22:16:18,214 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bd6592c37bb2:0-2 {event_type=RS_FLUSH_REGIONS, pid=138}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732486556329.744bbec6934be304e01b009be760323d. 2024-11-24T22:16:18,214 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bd6592c37bb2:0-2 {event_type=RS_FLUSH_REGIONS, pid=138}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=138 2024-11-24T22:16:18,215 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37701 {}] master.HMaster(4169): Remote procedure done, pid=138 2024-11-24T22:16:18,218 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=138, resume processing ppid=137 2024-11-24T22:16:18,218 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=138, ppid=137, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.3020 sec 2024-11-24T22:16:18,219 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=137, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=137, table=TestAcidGuarantees in 1.3070 sec 2024-11-24T22:16:18,425 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40087 {}] regionserver.HRegion(8855): Flush requested on 744bbec6934be304e01b009be760323d 2024-11-24T22:16:18,425 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 744bbec6934be304e01b009be760323d 3/3 column families, dataSize=80.51 KB heapSize=239.81 KB 2024-11-24T22:16:18,426 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 744bbec6934be304e01b009be760323d, store=A 2024-11-24T22:16:18,426 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-24T22:16:18,426 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 744bbec6934be304e01b009be760323d, store=B 2024-11-24T22:16:18,426 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-24T22:16:18,426 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 744bbec6934be304e01b009be760323d, store=C 2024-11-24T22:16:18,426 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-24T22:16:18,430 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/744bbec6934be304e01b009be760323d/.tmp/A/f23753f45d0b43bbb035eefc6588d590 is 50, key is test_row_0/A:col10/1732486577768/Put/seqid=0 2024-11-24T22:16:18,434 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45929 is added to blk_1073742371_1547 (size=12301) 2024-11-24T22:16:18,491 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40087 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=744bbec6934be304e01b009be760323d, server=bd6592c37bb2,40087,1732486423713 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-24T22:16:18,491 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40087 {}] ipc.CallRunner(138): callId: 137 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46842 deadline: 1732486638480, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=744bbec6934be304e01b009be760323d, server=bd6592c37bb2,40087,1732486423713 2024-11-24T22:16:18,492 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732486556329.744bbec6934be304e01b009be760323d., hostname=bd6592c37bb2,40087,1732486423713, seqNum=2 , the old value is region=TestAcidGuarantees,,1732486556329.744bbec6934be304e01b009be760323d., hostname=bd6592c37bb2,40087,1732486423713, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=744bbec6934be304e01b009be760323d, server=bd6592c37bb2,40087,1732486423713 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T22:16:18,492 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732486556329.744bbec6934be304e01b009be760323d., hostname=bd6592c37bb2,40087,1732486423713, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=744bbec6934be304e01b009be760323d, server=bd6592c37bb2,40087,1732486423713 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T22:16:18,492 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732486556329.744bbec6934be304e01b009be760323d., hostname=bd6592c37bb2,40087,1732486423713, seqNum=2 because the exception is null or not the one we care about 2024-11-24T22:16:18,494 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40087 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=744bbec6934be304e01b009be760323d, server=bd6592c37bb2,40087,1732486423713 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-24T22:16:18,494 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40087 {}] ipc.CallRunner(138): callId: 210 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46792 deadline: 1732486638482, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=744bbec6934be304e01b009be760323d, server=bd6592c37bb2,40087,1732486423713 2024-11-24T22:16:18,494 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732486556329.744bbec6934be304e01b009be760323d., hostname=bd6592c37bb2,40087,1732486423713, seqNum=2 , the old value is region=TestAcidGuarantees,,1732486556329.744bbec6934be304e01b009be760323d., hostname=bd6592c37bb2,40087,1732486423713, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=744bbec6934be304e01b009be760323d, server=bd6592c37bb2,40087,1732486423713 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T22:16:18,494 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732486556329.744bbec6934be304e01b009be760323d., hostname=bd6592c37bb2,40087,1732486423713, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=744bbec6934be304e01b009be760323d, server=bd6592c37bb2,40087,1732486423713 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T22:16:18,494 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732486556329.744bbec6934be304e01b009be760323d., hostname=bd6592c37bb2,40087,1732486423713, seqNum=2 because the exception is null or not the one we care about 2024-11-24T22:16:18,522 INFO [Thread-2041 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-2041. 2024-11-24T22:16:18,522 DEBUG [Thread-2041 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicScanReader.workDone(AcidGuaranteesTestTool.java:302) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-24T22:16:18,522 DEBUG [Thread-2041 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-24T22:16:18,522 DEBUG [Thread-2041 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-24T22:16:18,522 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-24T22:16:18,525 INFO [Thread-2049 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-2049. 2024-11-24T22:16:18,525 DEBUG [Thread-2049 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicScanReader.workDone(AcidGuaranteesTestTool.java:302) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-24T22:16:18,525 DEBUG [Thread-2049 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-24T22:16:18,525 DEBUG [Thread-2049 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-24T22:16:18,525 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-24T22:16:18,526 INFO [Thread-2045 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-2045. 2024-11-24T22:16:18,526 DEBUG [Thread-2045 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicScanReader.workDone(AcidGuaranteesTestTool.java:302) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-24T22:16:18,527 DEBUG [Thread-2045 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-24T22:16:18,527 DEBUG [Thread-2045 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-24T22:16:18,527 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-24T22:16:18,534 INFO [Thread-2043 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-2043. 2024-11-24T22:16:18,534 DEBUG [Thread-2043 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicScanReader.workDone(AcidGuaranteesTestTool.java:302) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-24T22:16:18,534 DEBUG [Thread-2043 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-24T22:16:18,534 DEBUG [Thread-2043 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-24T22:16:18,534 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-24T22:16:18,536 INFO [Thread-2047 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-2047. 2024-11-24T22:16:18,536 DEBUG [Thread-2047 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicScanReader.workDone(AcidGuaranteesTestTool.java:302) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-24T22:16:18,536 DEBUG [Thread-2047 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-24T22:16:18,536 DEBUG [Thread-2047 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-24T22:16:18,536 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-24T22:16:18,601 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40087 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=744bbec6934be304e01b009be760323d, server=bd6592c37bb2,40087,1732486423713 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-24T22:16:18,601 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40087 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=744bbec6934be304e01b009be760323d, server=bd6592c37bb2,40087,1732486423713 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-24T22:16:18,601 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40087 {}] ipc.CallRunner(138): callId: 138 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46842 deadline: 1732486638601, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=744bbec6934be304e01b009be760323d, server=bd6592c37bb2,40087,1732486423713 2024-11-24T22:16:18,601 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40087 {}] ipc.CallRunner(138): callId: 211 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46792 deadline: 1732486638601, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=744bbec6934be304e01b009be760323d, server=bd6592c37bb2,40087,1732486423713 2024-11-24T22:16:18,601 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732486556329.744bbec6934be304e01b009be760323d., hostname=bd6592c37bb2,40087,1732486423713, seqNum=2 , the old value is region=TestAcidGuarantees,,1732486556329.744bbec6934be304e01b009be760323d., hostname=bd6592c37bb2,40087,1732486423713, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=744bbec6934be304e01b009be760323d, server=bd6592c37bb2,40087,1732486423713 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T22:16:18,601 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732486556329.744bbec6934be304e01b009be760323d., hostname=bd6592c37bb2,40087,1732486423713, seqNum=2 , the old value is region=TestAcidGuarantees,,1732486556329.744bbec6934be304e01b009be760323d., hostname=bd6592c37bb2,40087,1732486423713, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=744bbec6934be304e01b009be760323d, server=bd6592c37bb2,40087,1732486423713 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T22:16:18,601 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732486556329.744bbec6934be304e01b009be760323d., hostname=bd6592c37bb2,40087,1732486423713, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=744bbec6934be304e01b009be760323d, server=bd6592c37bb2,40087,1732486423713 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T22:16:18,601 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732486556329.744bbec6934be304e01b009be760323d., hostname=bd6592c37bb2,40087,1732486423713, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=744bbec6934be304e01b009be760323d, server=bd6592c37bb2,40087,1732486423713 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T22:16:18,601 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732486556329.744bbec6934be304e01b009be760323d., hostname=bd6592c37bb2,40087,1732486423713, seqNum=2 because the exception is null or not the one we care about 2024-11-24T22:16:18,601 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732486556329.744bbec6934be304e01b009be760323d., hostname=bd6592c37bb2,40087,1732486423713, seqNum=2 because the exception is null or not the one we care about 2024-11-24T22:16:18,810 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40087 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=744bbec6934be304e01b009be760323d, server=bd6592c37bb2,40087,1732486423713 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-24T22:16:18,810 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40087 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=744bbec6934be304e01b009be760323d, server=bd6592c37bb2,40087,1732486423713 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-24T22:16:18,811 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40087 {}] ipc.CallRunner(138): callId: 139 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46842 deadline: 1732486638810, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=744bbec6934be304e01b009be760323d, server=bd6592c37bb2,40087,1732486423713 2024-11-24T22:16:18,811 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40087 {}] ipc.CallRunner(138): callId: 212 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46792 deadline: 1732486638810, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=744bbec6934be304e01b009be760323d, server=bd6592c37bb2,40087,1732486423713 2024-11-24T22:16:18,811 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732486556329.744bbec6934be304e01b009be760323d., hostname=bd6592c37bb2,40087,1732486423713, seqNum=2 , the old value is region=TestAcidGuarantees,,1732486556329.744bbec6934be304e01b009be760323d., hostname=bd6592c37bb2,40087,1732486423713, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=744bbec6934be304e01b009be760323d, server=bd6592c37bb2,40087,1732486423713 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T22:16:18,811 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732486556329.744bbec6934be304e01b009be760323d., hostname=bd6592c37bb2,40087,1732486423713, seqNum=2 , the old value is region=TestAcidGuarantees,,1732486556329.744bbec6934be304e01b009be760323d., hostname=bd6592c37bb2,40087,1732486423713, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=744bbec6934be304e01b009be760323d, server=bd6592c37bb2,40087,1732486423713 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T22:16:18,811 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732486556329.744bbec6934be304e01b009be760323d., hostname=bd6592c37bb2,40087,1732486423713, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=744bbec6934be304e01b009be760323d, server=bd6592c37bb2,40087,1732486423713 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T22:16:18,811 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732486556329.744bbec6934be304e01b009be760323d., hostname=bd6592c37bb2,40087,1732486423713, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=744bbec6934be304e01b009be760323d, server=bd6592c37bb2,40087,1732486423713 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T22:16:18,811 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732486556329.744bbec6934be304e01b009be760323d., hostname=bd6592c37bb2,40087,1732486423713, seqNum=2 because the exception is null or not the one we care about 2024-11-24T22:16:18,811 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732486556329.744bbec6934be304e01b009be760323d., hostname=bd6592c37bb2,40087,1732486423713, seqNum=2 because the exception is null or not the one we care about 2024-11-24T22:16:18,834 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=26.84 KB at sequenceid=430 (bloomFilter=true), to=hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/744bbec6934be304e01b009be760323d/.tmp/A/f23753f45d0b43bbb035eefc6588d590 2024-11-24T22:16:18,840 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/744bbec6934be304e01b009be760323d/.tmp/B/04d385c9cd224ba98ad89e3d72b81149 is 50, key is test_row_0/B:col10/1732486577768/Put/seqid=0 2024-11-24T22:16:18,842 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45929 is added to blk_1073742372_1548 (size=12301) 2024-11-24T22:16:19,061 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37701 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=137 2024-11-24T22:16:19,061 INFO [RPCClient-NioEventLoopGroup-4-10 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-24T22:16:19,121 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40087 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=744bbec6934be304e01b009be760323d, server=bd6592c37bb2,40087,1732486423713 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-24T22:16:19,121 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40087 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=744bbec6934be304e01b009be760323d, server=bd6592c37bb2,40087,1732486423713 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-24T22:16:19,121 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40087 {}] ipc.CallRunner(138): callId: 213 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46792 deadline: 1732486639121, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=744bbec6934be304e01b009be760323d, server=bd6592c37bb2,40087,1732486423713 2024-11-24T22:16:19,121 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40087 {}] ipc.CallRunner(138): callId: 140 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46842 deadline: 1732486639121, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=744bbec6934be304e01b009be760323d, server=bd6592c37bb2,40087,1732486423713 2024-11-24T22:16:19,121 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732486556329.744bbec6934be304e01b009be760323d., hostname=bd6592c37bb2,40087,1732486423713, seqNum=2 , the old value is region=TestAcidGuarantees,,1732486556329.744bbec6934be304e01b009be760323d., hostname=bd6592c37bb2,40087,1732486423713, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=744bbec6934be304e01b009be760323d, server=bd6592c37bb2,40087,1732486423713 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T22:16:19,121 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732486556329.744bbec6934be304e01b009be760323d., hostname=bd6592c37bb2,40087,1732486423713, seqNum=2 , the old value is region=TestAcidGuarantees,,1732486556329.744bbec6934be304e01b009be760323d., hostname=bd6592c37bb2,40087,1732486423713, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=744bbec6934be304e01b009be760323d, server=bd6592c37bb2,40087,1732486423713 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T22:16:19,121 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732486556329.744bbec6934be304e01b009be760323d., hostname=bd6592c37bb2,40087,1732486423713, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=744bbec6934be304e01b009be760323d, server=bd6592c37bb2,40087,1732486423713 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T22:16:19,121 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732486556329.744bbec6934be304e01b009be760323d., hostname=bd6592c37bb2,40087,1732486423713, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=744bbec6934be304e01b009be760323d, server=bd6592c37bb2,40087,1732486423713 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T22:16:19,121 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732486556329.744bbec6934be304e01b009be760323d., hostname=bd6592c37bb2,40087,1732486423713, seqNum=2 because the exception is null or not the one we care about 2024-11-24T22:16:19,121 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732486556329.744bbec6934be304e01b009be760323d., hostname=bd6592c37bb2,40087,1732486423713, seqNum=2 because the exception is null or not the one we care about 2024-11-24T22:16:19,243 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=26.84 KB at sequenceid=430 (bloomFilter=true), to=hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/744bbec6934be304e01b009be760323d/.tmp/B/04d385c9cd224ba98ad89e3d72b81149 2024-11-24T22:16:19,248 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/744bbec6934be304e01b009be760323d/.tmp/C/929ecd28586e42e1ab4a45d46f0de599 is 50, key is test_row_0/C:col10/1732486577768/Put/seqid=0 2024-11-24T22:16:19,251 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45929 is added to blk_1073742373_1549 (size=12301) 2024-11-24T22:16:19,631 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40087 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=744bbec6934be304e01b009be760323d, server=bd6592c37bb2,40087,1732486423713 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-24T22:16:19,631 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40087 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=744bbec6934be304e01b009be760323d, server=bd6592c37bb2,40087,1732486423713 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-24T22:16:19,631 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40087 {}] ipc.CallRunner(138): callId: 214 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46792 deadline: 1732486639631, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=744bbec6934be304e01b009be760323d, server=bd6592c37bb2,40087,1732486423713 2024-11-24T22:16:19,631 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40087 {}] ipc.CallRunner(138): callId: 141 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46842 deadline: 1732486639631, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=744bbec6934be304e01b009be760323d, server=bd6592c37bb2,40087,1732486423713 2024-11-24T22:16:19,632 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732486556329.744bbec6934be304e01b009be760323d., hostname=bd6592c37bb2,40087,1732486423713, seqNum=2 , the old value is region=TestAcidGuarantees,,1732486556329.744bbec6934be304e01b009be760323d., hostname=bd6592c37bb2,40087,1732486423713, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=744bbec6934be304e01b009be760323d, server=bd6592c37bb2,40087,1732486423713 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T22:16:19,632 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732486556329.744bbec6934be304e01b009be760323d., hostname=bd6592c37bb2,40087,1732486423713, seqNum=2 , the old value is region=TestAcidGuarantees,,1732486556329.744bbec6934be304e01b009be760323d., hostname=bd6592c37bb2,40087,1732486423713, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=744bbec6934be304e01b009be760323d, server=bd6592c37bb2,40087,1732486423713 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T22:16:19,632 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732486556329.744bbec6934be304e01b009be760323d., hostname=bd6592c37bb2,40087,1732486423713, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=744bbec6934be304e01b009be760323d, server=bd6592c37bb2,40087,1732486423713 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T22:16:19,632 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732486556329.744bbec6934be304e01b009be760323d., hostname=bd6592c37bb2,40087,1732486423713, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=744bbec6934be304e01b009be760323d, server=bd6592c37bb2,40087,1732486423713 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T22:16:19,632 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732486556329.744bbec6934be304e01b009be760323d., hostname=bd6592c37bb2,40087,1732486423713, seqNum=2 because the exception is null or not the one we care about 2024-11-24T22:16:19,632 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732486556329.744bbec6934be304e01b009be760323d., hostname=bd6592c37bb2,40087,1732486423713, seqNum=2 because the exception is null or not the one we care about 2024-11-24T22:16:19,651 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=26.84 KB at sequenceid=430 (bloomFilter=true), to=hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/744bbec6934be304e01b009be760323d/.tmp/C/929ecd28586e42e1ab4a45d46f0de599 2024-11-24T22:16:19,654 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/744bbec6934be304e01b009be760323d/.tmp/A/f23753f45d0b43bbb035eefc6588d590 as hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/744bbec6934be304e01b009be760323d/A/f23753f45d0b43bbb035eefc6588d590 2024-11-24T22:16:19,657 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/744bbec6934be304e01b009be760323d/A/f23753f45d0b43bbb035eefc6588d590, entries=150, sequenceid=430, filesize=12.0 K 2024-11-24T22:16:19,657 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/744bbec6934be304e01b009be760323d/.tmp/B/04d385c9cd224ba98ad89e3d72b81149 as hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/744bbec6934be304e01b009be760323d/B/04d385c9cd224ba98ad89e3d72b81149 2024-11-24T22:16:19,660 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/744bbec6934be304e01b009be760323d/B/04d385c9cd224ba98ad89e3d72b81149, entries=150, sequenceid=430, filesize=12.0 K 2024-11-24T22:16:19,660 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/744bbec6934be304e01b009be760323d/.tmp/C/929ecd28586e42e1ab4a45d46f0de599 as hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/744bbec6934be304e01b009be760323d/C/929ecd28586e42e1ab4a45d46f0de599 2024-11-24T22:16:19,663 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/744bbec6934be304e01b009be760323d/C/929ecd28586e42e1ab4a45d46f0de599, entries=150, sequenceid=430, filesize=12.0 K 2024-11-24T22:16:19,663 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~80.51 KB/82440, heapSize ~239.77 KB/245520, currentSize=93.93 KB/96180 for 744bbec6934be304e01b009be760323d in 1238ms, sequenceid=430, compaction requested=true 2024-11-24T22:16:19,663 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 744bbec6934be304e01b009be760323d: 2024-11-24T22:16:19,663 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 744bbec6934be304e01b009be760323d:A, priority=-2147483648, current under compaction store size is 1 2024-11-24T22:16:19,663 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-24T22:16:19,663 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 744bbec6934be304e01b009be760323d:B, priority=-2147483648, current under compaction store size is 2 2024-11-24T22:16:19,663 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-24T22:16:19,663 DEBUG [RS:0;bd6592c37bb2:40087-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-24T22:16:19,663 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 744bbec6934be304e01b009be760323d:C, priority=-2147483648, current under compaction store size is 3 2024-11-24T22:16:19,663 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-24T22:16:19,663 DEBUG [RS:0;bd6592c37bb2:40087-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-24T22:16:19,664 DEBUG [RS:0;bd6592c37bb2:40087-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 50158 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-24T22:16:19,664 DEBUG [RS:0;bd6592c37bb2:40087-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 57478 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-24T22:16:19,664 DEBUG [RS:0;bd6592c37bb2:40087-shortCompactions-0 {}] regionserver.HStore(1541): 744bbec6934be304e01b009be760323d/A is initiating minor compaction (all files) 2024-11-24T22:16:19,664 DEBUG [RS:0;bd6592c37bb2:40087-longCompactions-0 {}] regionserver.HStore(1541): 744bbec6934be304e01b009be760323d/B is initiating minor compaction (all files) 2024-11-24T22:16:19,664 INFO [RS:0;bd6592c37bb2:40087-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 744bbec6934be304e01b009be760323d/B in TestAcidGuarantees,,1732486556329.744bbec6934be304e01b009be760323d. 2024-11-24T22:16:19,664 INFO [RS:0;bd6592c37bb2:40087-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 744bbec6934be304e01b009be760323d/A in TestAcidGuarantees,,1732486556329.744bbec6934be304e01b009be760323d. 2024-11-24T22:16:19,664 INFO [RS:0;bd6592c37bb2:40087-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/744bbec6934be304e01b009be760323d/B/015f0d78f00e465198f4def8f41b689f, hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/744bbec6934be304e01b009be760323d/B/442e65fffb94469f8109f1c971c49198, hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/744bbec6934be304e01b009be760323d/B/b8f9a37fd50c47c0b695ccbd0b13417b, hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/744bbec6934be304e01b009be760323d/B/04d385c9cd224ba98ad89e3d72b81149] into tmpdir=hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/744bbec6934be304e01b009be760323d/.tmp, totalSize=49.0 K 2024-11-24T22:16:19,664 INFO [RS:0;bd6592c37bb2:40087-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/744bbec6934be304e01b009be760323d/A/8939132dcea842a3bfda12ce12488284, hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/744bbec6934be304e01b009be760323d/A/bae78c8dcb824903b1bac1b8970098b3, hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/744bbec6934be304e01b009be760323d/A/ce29d063c8214791bbc3b99864b9d49e, hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/744bbec6934be304e01b009be760323d/A/f23753f45d0b43bbb035eefc6588d590] into tmpdir=hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/744bbec6934be304e01b009be760323d/.tmp, totalSize=56.1 K 2024-11-24T22:16:19,665 DEBUG [RS:0;bd6592c37bb2:40087-longCompactions-0 {}] compactions.Compactor(225): Compacting 015f0d78f00e465198f4def8f41b689f, keycount=150, bloomtype=ROW, size=12.9 K, encoding=NONE, compression=NONE, seqNum=380, earliestPutTs=1732486576048 2024-11-24T22:16:19,665 DEBUG [RS:0;bd6592c37bb2:40087-shortCompactions-0 {}] compactions.Compactor(225): Compacting 8939132dcea842a3bfda12ce12488284, keycount=150, bloomtype=ROW, size=12.9 K, encoding=NONE, compression=NONE, seqNum=380, earliestPutTs=1732486576048 2024-11-24T22:16:19,665 DEBUG [RS:0;bd6592c37bb2:40087-longCompactions-0 {}] compactions.Compactor(225): Compacting 442e65fffb94469f8109f1c971c49198, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=394, earliestPutTs=1732486576432 2024-11-24T22:16:19,665 DEBUG [RS:0;bd6592c37bb2:40087-shortCompactions-0 {}] compactions.Compactor(225): Compacting bae78c8dcb824903b1bac1b8970098b3, keycount=300, bloomtype=ROW, size=19.2 K, encoding=NONE, compression=NONE, seqNum=394, earliestPutTs=1732486576432 2024-11-24T22:16:19,665 DEBUG [RS:0;bd6592c37bb2:40087-longCompactions-0 {}] compactions.Compactor(225): Compacting b8f9a37fd50c47c0b695ccbd0b13417b, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=415, earliestPutTs=1732486576538 2024-11-24T22:16:19,665 DEBUG [RS:0;bd6592c37bb2:40087-shortCompactions-0 {}] compactions.Compactor(225): Compacting ce29d063c8214791bbc3b99864b9d49e, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=415, earliestPutTs=1732486576538 2024-11-24T22:16:19,665 DEBUG [RS:0;bd6592c37bb2:40087-longCompactions-0 {}] compactions.Compactor(225): Compacting 04d385c9cd224ba98ad89e3d72b81149, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=430, earliestPutTs=1732486577743 2024-11-24T22:16:19,665 DEBUG [RS:0;bd6592c37bb2:40087-shortCompactions-0 {}] compactions.Compactor(225): Compacting f23753f45d0b43bbb035eefc6588d590, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=430, earliestPutTs=1732486577743 2024-11-24T22:16:19,671 INFO [RS:0;bd6592c37bb2:40087-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 744bbec6934be304e01b009be760323d#A#compaction#470 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 57.69 MB/second 2024-11-24T22:16:19,671 INFO [RS:0;bd6592c37bb2:40087-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 744bbec6934be304e01b009be760323d#B#compaction#471 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 57.69 MB/second 2024-11-24T22:16:19,672 DEBUG [RS:0;bd6592c37bb2:40087-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/744bbec6934be304e01b009be760323d/.tmp/B/a196a970e0dc4c8a965203b83b106cbf is 50, key is test_row_0/B:col10/1732486577768/Put/seqid=0 2024-11-24T22:16:19,672 DEBUG [RS:0;bd6592c37bb2:40087-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/744bbec6934be304e01b009be760323d/.tmp/A/2fd557a0f7c34347b9cbf6d42ee42a09 is 50, key is test_row_0/A:col10/1732486577768/Put/seqid=0 2024-11-24T22:16:19,674 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45929 is added to blk_1073742375_1551 (size=13391) 2024-11-24T22:16:19,675 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45929 is added to blk_1073742374_1550 (size=13391) 2024-11-24T22:16:20,078 DEBUG [RS:0;bd6592c37bb2:40087-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/744bbec6934be304e01b009be760323d/.tmp/B/a196a970e0dc4c8a965203b83b106cbf as hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/744bbec6934be304e01b009be760323d/B/a196a970e0dc4c8a965203b83b106cbf 2024-11-24T22:16:20,078 DEBUG [RS:0;bd6592c37bb2:40087-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/744bbec6934be304e01b009be760323d/.tmp/A/2fd557a0f7c34347b9cbf6d42ee42a09 as hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/744bbec6934be304e01b009be760323d/A/2fd557a0f7c34347b9cbf6d42ee42a09 2024-11-24T22:16:20,082 INFO [RS:0;bd6592c37bb2:40087-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 744bbec6934be304e01b009be760323d/B of 744bbec6934be304e01b009be760323d into a196a970e0dc4c8a965203b83b106cbf(size=13.1 K), total size for store is 13.1 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-24T22:16:20,082 DEBUG [RS:0;bd6592c37bb2:40087-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 744bbec6934be304e01b009be760323d: 2024-11-24T22:16:20,082 INFO [RS:0;bd6592c37bb2:40087-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732486556329.744bbec6934be304e01b009be760323d., storeName=744bbec6934be304e01b009be760323d/B, priority=12, startTime=1732486579663; duration=0sec 2024-11-24T22:16:20,082 DEBUG [RS:0;bd6592c37bb2:40087-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-24T22:16:20,082 DEBUG [RS:0;bd6592c37bb2:40087-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 744bbec6934be304e01b009be760323d:B 2024-11-24T22:16:20,082 INFO [RS:0;bd6592c37bb2:40087-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 744bbec6934be304e01b009be760323d/A of 744bbec6934be304e01b009be760323d into 2fd557a0f7c34347b9cbf6d42ee42a09(size=13.1 K), total size for store is 13.1 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-24T22:16:20,082 DEBUG [RS:0;bd6592c37bb2:40087-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-24T22:16:20,082 DEBUG [RS:0;bd6592c37bb2:40087-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 744bbec6934be304e01b009be760323d: 2024-11-24T22:16:20,082 INFO [RS:0;bd6592c37bb2:40087-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732486556329.744bbec6934be304e01b009be760323d., storeName=744bbec6934be304e01b009be760323d/A, priority=12, startTime=1732486579663; duration=0sec 2024-11-24T22:16:20,082 DEBUG [RS:0;bd6592c37bb2:40087-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-24T22:16:20,082 DEBUG [RS:0;bd6592c37bb2:40087-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 744bbec6934be304e01b009be760323d:A 2024-11-24T22:16:20,083 DEBUG [RS:0;bd6592c37bb2:40087-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 50158 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-24T22:16:20,083 DEBUG [RS:0;bd6592c37bb2:40087-longCompactions-0 {}] regionserver.HStore(1541): 744bbec6934be304e01b009be760323d/C is initiating minor compaction (all files) 2024-11-24T22:16:20,083 INFO [RS:0;bd6592c37bb2:40087-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 744bbec6934be304e01b009be760323d/C in TestAcidGuarantees,,1732486556329.744bbec6934be304e01b009be760323d. 2024-11-24T22:16:20,083 INFO [RS:0;bd6592c37bb2:40087-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/744bbec6934be304e01b009be760323d/C/05ec45638c4643b1b3313cdf31e2f636, hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/744bbec6934be304e01b009be760323d/C/f67ba0237d4c48d8b9ce5f846119b2ba, hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/744bbec6934be304e01b009be760323d/C/7b7ae2272b0f466688e0574d67613286, hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/744bbec6934be304e01b009be760323d/C/929ecd28586e42e1ab4a45d46f0de599] into tmpdir=hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/744bbec6934be304e01b009be760323d/.tmp, totalSize=49.0 K 2024-11-24T22:16:20,083 DEBUG [RS:0;bd6592c37bb2:40087-longCompactions-0 {}] compactions.Compactor(225): Compacting 05ec45638c4643b1b3313cdf31e2f636, keycount=150, bloomtype=ROW, size=12.9 K, encoding=NONE, compression=NONE, seqNum=380, earliestPutTs=1732486576048 2024-11-24T22:16:20,083 DEBUG [RS:0;bd6592c37bb2:40087-longCompactions-0 {}] compactions.Compactor(225): Compacting f67ba0237d4c48d8b9ce5f846119b2ba, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=394, earliestPutTs=1732486576432 2024-11-24T22:16:20,083 DEBUG [RS:0;bd6592c37bb2:40087-longCompactions-0 {}] compactions.Compactor(225): Compacting 7b7ae2272b0f466688e0574d67613286, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=415, earliestPutTs=1732486576538 2024-11-24T22:16:20,084 DEBUG [RS:0;bd6592c37bb2:40087-longCompactions-0 {}] compactions.Compactor(225): Compacting 929ecd28586e42e1ab4a45d46f0de599, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=430, earliestPutTs=1732486577743 2024-11-24T22:16:20,090 INFO [RS:0;bd6592c37bb2:40087-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 744bbec6934be304e01b009be760323d#C#compaction#472 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 57.69 MB/second 2024-11-24T22:16:20,090 DEBUG [RS:0;bd6592c37bb2:40087-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/744bbec6934be304e01b009be760323d/.tmp/C/365e82b4a9704352abd426389dcfbb91 is 50, key is test_row_0/C:col10/1732486577768/Put/seqid=0 2024-11-24T22:16:20,093 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45929 is added to blk_1073742376_1552 (size=13391) 2024-11-24T22:16:20,497 DEBUG [RS:0;bd6592c37bb2:40087-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/744bbec6934be304e01b009be760323d/.tmp/C/365e82b4a9704352abd426389dcfbb91 as hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/744bbec6934be304e01b009be760323d/C/365e82b4a9704352abd426389dcfbb91 2024-11-24T22:16:20,500 INFO [RS:0;bd6592c37bb2:40087-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 744bbec6934be304e01b009be760323d/C of 744bbec6934be304e01b009be760323d into 365e82b4a9704352abd426389dcfbb91(size=13.1 K), total size for store is 13.1 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-24T22:16:20,500 DEBUG [RS:0;bd6592c37bb2:40087-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 744bbec6934be304e01b009be760323d: 2024-11-24T22:16:20,500 INFO [RS:0;bd6592c37bb2:40087-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732486556329.744bbec6934be304e01b009be760323d., storeName=744bbec6934be304e01b009be760323d/C, priority=12, startTime=1732486579663; duration=0sec 2024-11-24T22:16:20,500 DEBUG [RS:0;bd6592c37bb2:40087-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-24T22:16:20,500 DEBUG [RS:0;bd6592c37bb2:40087-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 744bbec6934be304e01b009be760323d:C 2024-11-24T22:16:20,642 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40087 {}] regionserver.HRegion(8855): Flush requested on 744bbec6934be304e01b009be760323d 2024-11-24T22:16:20,642 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 744bbec6934be304e01b009be760323d 3/3 column families, dataSize=107.34 KB heapSize=319.50 KB 2024-11-24T22:16:20,642 INFO [Thread-2030 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-2030. 2024-11-24T22:16:20,642 DEBUG [Thread-2030 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-24T22:16:20,642 INFO [Thread-2036 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-2036. 2024-11-24T22:16:20,642 DEBUG [Thread-2030 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-24T22:16:20,642 DEBUG [Thread-2036 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-24T22:16:20,642 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 744bbec6934be304e01b009be760323d, store=A 2024-11-24T22:16:20,642 DEBUG [Thread-2036 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-24T22:16:20,642 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-24T22:16:20,643 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 744bbec6934be304e01b009be760323d, store=B 2024-11-24T22:16:20,643 DEBUG [Thread-2030 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-24T22:16:20,643 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-24T22:16:20,643 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 744bbec6934be304e01b009be760323d, store=C 2024-11-24T22:16:20,643 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-24T22:16:20,643 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-24T22:16:20,643 DEBUG [Thread-2036 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-24T22:16:20,643 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-24T22:16:20,646 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/744bbec6934be304e01b009be760323d/.tmp/A/0d73e3a2e443431baf4d1b1894f7323a is 50, key is test_row_0/A:col10/1732486578455/Put/seqid=0 2024-11-24T22:16:20,649 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45929 is added to blk_1073742377_1553 (size=12301) 2024-11-24T22:16:21,022 INFO [Thread-2038 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-2038. 2024-11-24T22:16:21,022 DEBUG [Thread-2038 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-24T22:16:21,022 DEBUG [Thread-2038 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-24T22:16:21,022 DEBUG [Thread-2038 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-24T22:16:21,022 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-24T22:16:21,032 INFO [Thread-2032 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-2032. 2024-11-24T22:16:21,032 DEBUG [Thread-2032 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-24T22:16:21,032 DEBUG [Thread-2032 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-24T22:16:21,032 DEBUG [Thread-2032 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-24T22:16:21,032 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-24T22:16:21,049 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=35.78 KB at sequenceid=452 (bloomFilter=true), to=hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/744bbec6934be304e01b009be760323d/.tmp/A/0d73e3a2e443431baf4d1b1894f7323a 2024-11-24T22:16:21,052 INFO [Thread-2034 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-2034. 2024-11-24T22:16:21,052 DEBUG [Thread-2034 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-24T22:16:21,052 DEBUG [Thread-2034 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-24T22:16:21,052 DEBUG [Thread-2034 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-24T22:16:21,052 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-24T22:16:21,052 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(392): Finished test. Writers: 2024-11-24T22:16:21,052 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 152 2024-11-24T22:16:21,052 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 34 2024-11-24T22:16:21,052 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 43 2024-11-24T22:16:21,052 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 88 2024-11-24T22:16:21,052 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 35 2024-11-24T22:16:21,052 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(396): Readers: 2024-11-24T22:16:21,052 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(400): Scanners: 2024-11-24T22:16:21,053 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(402): scanned 1020 2024-11-24T22:16:21,053 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(403): verified 3050 rows 2024-11-24T22:16:21,053 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(402): scanned 1019 2024-11-24T22:16:21,053 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(403): verified 3057 rows 2024-11-24T22:16:21,053 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(402): scanned 1024 2024-11-24T22:16:21,053 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(403): verified 3064 rows 2024-11-24T22:16:21,053 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(402): scanned 1017 2024-11-24T22:16:21,053 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(403): verified 3051 rows 2024-11-24T22:16:21,053 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(402): scanned 1019 2024-11-24T22:16:21,053 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(403): verified 3054 rows 2024-11-24T22:16:21,053 INFO [Time-limited test {}] client.AsyncConnectionImpl(233): Connection has been closed by Time-limited test. 2024-11-24T22:16:21,053 DEBUG [Time-limited test {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool.doWork(AcidGuaranteesTestTool.java:131) at org.apache.hadoop.hbase.util.AbstractHBaseTool.run(AbstractHBaseTool.java:150) at org.apache.hadoop.hbase.AcidGuaranteesTestBase.runTestAtomicity(AcidGuaranteesTestBase.java:102) at org.apache.hadoop.hbase.AcidGuaranteesTestBase.runTestAtomicity(AcidGuaranteesTestBase.java:91) at org.apache.hadoop.hbase.AcidGuaranteesTestBase.testScanAtomicity(AcidGuaranteesTestBase.java:112) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) at java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) at java.base/java.lang.reflect.Method.invoke(Method.java:568) at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) at java.base/java.util.concurrent.FutureTask.run(FutureTask.java:264) at java.base/java.lang.Thread.run(Thread.java:840) 2024-11-24T22:16:21,053 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-24T22:16:21,053 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-24T22:16:21,053 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-24T22:16:21,054 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37701 {}] master.HMaster$13(2820): Client=jenkins//172.17.0.2 disable TestAcidGuarantees 2024-11-24T22:16:21,054 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37701 {}] procedure2.ProcedureExecutor(1139): Stored pid=139, state=RUNNABLE:DISABLE_TABLE_PREPARE, hasLock=false; DisableTableProcedure table=TestAcidGuarantees 2024-11-24T22:16:21,055 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/744bbec6934be304e01b009be760323d/.tmp/B/2aeb57cd3bf9441bb1932d9dcbba2ec2 is 50, key is test_row_0/B:col10/1732486578455/Put/seqid=0 2024-11-24T22:16:21,056 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37701 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=139 2024-11-24T22:16:21,056 DEBUG [PEWorker-3 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1732486581056"}]},"ts":"1732486581056"} 2024-11-24T22:16:21,057 INFO [PEWorker-3 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=DISABLING in hbase:meta 2024-11-24T22:16:21,057 INFO [PEWorker-3 {}] procedure.DisableTableProcedure(284): Set TestAcidGuarantees to state=DISABLING 2024-11-24T22:16:21,058 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=140, ppid=139, state=RUNNABLE:CLOSE_TABLE_REGIONS_SCHEDULE, hasLock=false; CloseTableRegionsProcedure table=TestAcidGuarantees}] 2024-11-24T22:16:21,058 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45929 is added to blk_1073742378_1554 (size=12301) 2024-11-24T22:16:21,059 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=141, ppid=140, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=744bbec6934be304e01b009be760323d, UNASSIGN}] 2024-11-24T22:16:21,060 INFO [PEWorker-2 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=141, ppid=140, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=744bbec6934be304e01b009be760323d, UNASSIGN 2024-11-24T22:16:21,060 INFO [PEWorker-2 {}] assignment.RegionStateStore(223): pid=141 updating hbase:meta row=744bbec6934be304e01b009be760323d, regionState=CLOSING, regionLocation=bd6592c37bb2,40087,1732486423713 2024-11-24T22:16:21,061 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=141, ppid=140, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=744bbec6934be304e01b009be760323d, UNASSIGN because future has completed 2024-11-24T22:16:21,061 DEBUG [PEWorker-4 {}] assignment.TransitRegionStateProcedure(375): Close region: isSplit: false: evictOnSplit: true: evictOnClose: false 2024-11-24T22:16:21,061 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=142, ppid=141, state=RUNNABLE, hasLock=false; CloseRegionProcedure 744bbec6934be304e01b009be760323d, server=bd6592c37bb2,40087,1732486423713}] 2024-11-24T22:16:21,161 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37701 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=139 2024-11-24T22:16:21,213 INFO [RS_CLOSE_REGION-regionserver/bd6592c37bb2:0-0 {event_type=M_RS_CLOSE_REGION, pid=142}] handler.UnassignRegionHandler(122): Close 744bbec6934be304e01b009be760323d 2024-11-24T22:16:21,213 DEBUG [RS_CLOSE_REGION-regionserver/bd6592c37bb2:0-0 {event_type=M_RS_CLOSE_REGION, pid=142}] handler.UnassignRegionHandler(136): Unassign region: split region: false: evictCache: false 2024-11-24T22:16:21,213 DEBUG [RS_CLOSE_REGION-regionserver/bd6592c37bb2:0-0 {event_type=M_RS_CLOSE_REGION, pid=142}] regionserver.HRegion(1722): Closing 744bbec6934be304e01b009be760323d, disabling compactions & flushes 2024-11-24T22:16:21,213 DEBUG [RS_CLOSE_REGION-regionserver/bd6592c37bb2:0-0 {event_type=M_RS_CLOSE_REGION, pid=142}] regionserver.HRegion(1993): waiting for 0 compactions & cache flush to complete for region TestAcidGuarantees,,1732486556329.744bbec6934be304e01b009be760323d. 2024-11-24T22:16:21,371 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37701 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=139 2024-11-24T22:16:21,459 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=35.78 KB at sequenceid=452 (bloomFilter=true), to=hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/744bbec6934be304e01b009be760323d/.tmp/B/2aeb57cd3bf9441bb1932d9dcbba2ec2 2024-11-24T22:16:21,464 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/744bbec6934be304e01b009be760323d/.tmp/C/80d409804e4a40d7a81a05a86f1b0f7f is 50, key is test_row_0/C:col10/1732486578455/Put/seqid=0 2024-11-24T22:16:21,471 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45929 is added to blk_1073742379_1555 (size=12301) 2024-11-24T22:16:21,681 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37701 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=139 2024-11-24T22:16:21,872 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=35.78 KB at sequenceid=452 (bloomFilter=true), to=hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/744bbec6934be304e01b009be760323d/.tmp/C/80d409804e4a40d7a81a05a86f1b0f7f 2024-11-24T22:16:21,875 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/744bbec6934be304e01b009be760323d/.tmp/A/0d73e3a2e443431baf4d1b1894f7323a as hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/744bbec6934be304e01b009be760323d/A/0d73e3a2e443431baf4d1b1894f7323a 2024-11-24T22:16:21,878 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/744bbec6934be304e01b009be760323d/A/0d73e3a2e443431baf4d1b1894f7323a, entries=150, sequenceid=452, filesize=12.0 K 2024-11-24T22:16:21,878 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/744bbec6934be304e01b009be760323d/.tmp/B/2aeb57cd3bf9441bb1932d9dcbba2ec2 as hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/744bbec6934be304e01b009be760323d/B/2aeb57cd3bf9441bb1932d9dcbba2ec2 2024-11-24T22:16:21,880 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/744bbec6934be304e01b009be760323d/B/2aeb57cd3bf9441bb1932d9dcbba2ec2, entries=150, sequenceid=452, filesize=12.0 K 2024-11-24T22:16:21,881 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/744bbec6934be304e01b009be760323d/.tmp/C/80d409804e4a40d7a81a05a86f1b0f7f as hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/744bbec6934be304e01b009be760323d/C/80d409804e4a40d7a81a05a86f1b0f7f 2024-11-24T22:16:21,883 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/744bbec6934be304e01b009be760323d/C/80d409804e4a40d7a81a05a86f1b0f7f, entries=150, sequenceid=452, filesize=12.0 K 2024-11-24T22:16:21,884 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~107.34 KB/109920, heapSize ~319.45 KB/327120, currentSize=20.13 KB/20610 for 744bbec6934be304e01b009be760323d in 1242ms, sequenceid=452, compaction requested=false 2024-11-24T22:16:21,884 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 744bbec6934be304e01b009be760323d: 2024-11-24T22:16:21,884 INFO [RS_CLOSE_REGION-regionserver/bd6592c37bb2:0-0 {event_type=M_RS_CLOSE_REGION, pid=142}] regionserver.HRegion(1755): Closing region TestAcidGuarantees,,1732486556329.744bbec6934be304e01b009be760323d. 2024-11-24T22:16:21,884 DEBUG [RS_CLOSE_REGION-regionserver/bd6592c37bb2:0-0 {event_type=M_RS_CLOSE_REGION, pid=142}] regionserver.HRegion(1776): Time limited wait for close lock on TestAcidGuarantees,,1732486556329.744bbec6934be304e01b009be760323d. 2024-11-24T22:16:21,884 DEBUG [RS_CLOSE_REGION-regionserver/bd6592c37bb2:0-0 {event_type=M_RS_CLOSE_REGION, pid=142}] regionserver.HRegion(1843): Acquired close lock on TestAcidGuarantees,,1732486556329.744bbec6934be304e01b009be760323d. after waiting 0 ms 2024-11-24T22:16:21,884 DEBUG [RS_CLOSE_REGION-regionserver/bd6592c37bb2:0-0 {event_type=M_RS_CLOSE_REGION, pid=142}] regionserver.HRegion(1853): Updates disabled for region TestAcidGuarantees,,1732486556329.744bbec6934be304e01b009be760323d. 2024-11-24T22:16:21,884 INFO [RS_CLOSE_REGION-regionserver/bd6592c37bb2:0-0 {event_type=M_RS_CLOSE_REGION, pid=142}] regionserver.HRegion(2902): Flushing 744bbec6934be304e01b009be760323d 3/3 column families, dataSize=20.13 KB heapSize=60.52 KB 2024-11-24T22:16:21,884 DEBUG [RS_CLOSE_REGION-regionserver/bd6592c37bb2:0-0 {event_type=M_RS_CLOSE_REGION, pid=142}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 744bbec6934be304e01b009be760323d, store=A 2024-11-24T22:16:21,884 DEBUG [RS_CLOSE_REGION-regionserver/bd6592c37bb2:0-0 {event_type=M_RS_CLOSE_REGION, pid=142}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-24T22:16:21,884 DEBUG [RS_CLOSE_REGION-regionserver/bd6592c37bb2:0-0 {event_type=M_RS_CLOSE_REGION, pid=142}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 744bbec6934be304e01b009be760323d, store=B 2024-11-24T22:16:21,884 DEBUG [RS_CLOSE_REGION-regionserver/bd6592c37bb2:0-0 {event_type=M_RS_CLOSE_REGION, pid=142}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-24T22:16:21,884 DEBUG [RS_CLOSE_REGION-regionserver/bd6592c37bb2:0-0 {event_type=M_RS_CLOSE_REGION, pid=142}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 744bbec6934be304e01b009be760323d, store=C 2024-11-24T22:16:21,884 DEBUG [RS_CLOSE_REGION-regionserver/bd6592c37bb2:0-0 {event_type=M_RS_CLOSE_REGION, pid=142}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-24T22:16:21,887 DEBUG [RS_CLOSE_REGION-regionserver/bd6592c37bb2:0-0 {event_type=M_RS_CLOSE_REGION, pid=142}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/744bbec6934be304e01b009be760323d/.tmp/A/21c5db2fe2e144398f211f2f114de18f is 50, key is test_row_0/A:col10/1732486581020/Put/seqid=0 2024-11-24T22:16:21,890 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45929 is added to blk_1073742380_1556 (size=9857) 2024-11-24T22:16:22,190 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37701 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=139 2024-11-24T22:16:22,291 INFO [RS_CLOSE_REGION-regionserver/bd6592c37bb2:0-0 {event_type=M_RS_CLOSE_REGION, pid=142}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=6.71 KB at sequenceid=458 (bloomFilter=true), to=hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/744bbec6934be304e01b009be760323d/.tmp/A/21c5db2fe2e144398f211f2f114de18f 2024-11-24T22:16:22,296 DEBUG [RS_CLOSE_REGION-regionserver/bd6592c37bb2:0-0 {event_type=M_RS_CLOSE_REGION, pid=142}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/744bbec6934be304e01b009be760323d/.tmp/B/ec23eec7790943728319381a913cd599 is 50, key is test_row_0/B:col10/1732486581020/Put/seqid=0 2024-11-24T22:16:22,298 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45929 is added to blk_1073742381_1557 (size=9857) 2024-11-24T22:16:22,699 INFO [RS_CLOSE_REGION-regionserver/bd6592c37bb2:0-0 {event_type=M_RS_CLOSE_REGION, pid=142}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=6.71 KB at sequenceid=458 (bloomFilter=true), to=hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/744bbec6934be304e01b009be760323d/.tmp/B/ec23eec7790943728319381a913cd599 2024-11-24T22:16:22,704 DEBUG [RS_CLOSE_REGION-regionserver/bd6592c37bb2:0-0 {event_type=M_RS_CLOSE_REGION, pid=142}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/744bbec6934be304e01b009be760323d/.tmp/C/ab4e14cb47dd453a8f163f214c641153 is 50, key is test_row_0/C:col10/1732486581020/Put/seqid=0 2024-11-24T22:16:22,707 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45929 is added to blk_1073742382_1558 (size=9857) 2024-11-24T22:16:23,107 INFO [RS_CLOSE_REGION-regionserver/bd6592c37bb2:0-0 {event_type=M_RS_CLOSE_REGION, pid=142}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=6.71 KB at sequenceid=458 (bloomFilter=true), to=hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/744bbec6934be304e01b009be760323d/.tmp/C/ab4e14cb47dd453a8f163f214c641153 2024-11-24T22:16:23,111 DEBUG [RS_CLOSE_REGION-regionserver/bd6592c37bb2:0-0 {event_type=M_RS_CLOSE_REGION, pid=142}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/744bbec6934be304e01b009be760323d/.tmp/A/21c5db2fe2e144398f211f2f114de18f as hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/744bbec6934be304e01b009be760323d/A/21c5db2fe2e144398f211f2f114de18f 2024-11-24T22:16:23,113 INFO [RS_CLOSE_REGION-regionserver/bd6592c37bb2:0-0 {event_type=M_RS_CLOSE_REGION, pid=142}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/744bbec6934be304e01b009be760323d/A/21c5db2fe2e144398f211f2f114de18f, entries=100, sequenceid=458, filesize=9.6 K 2024-11-24T22:16:23,113 DEBUG [RS_CLOSE_REGION-regionserver/bd6592c37bb2:0-0 {event_type=M_RS_CLOSE_REGION, pid=142}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/744bbec6934be304e01b009be760323d/.tmp/B/ec23eec7790943728319381a913cd599 as hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/744bbec6934be304e01b009be760323d/B/ec23eec7790943728319381a913cd599 2024-11-24T22:16:23,116 INFO [RS_CLOSE_REGION-regionserver/bd6592c37bb2:0-0 {event_type=M_RS_CLOSE_REGION, pid=142}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/744bbec6934be304e01b009be760323d/B/ec23eec7790943728319381a913cd599, entries=100, sequenceid=458, filesize=9.6 K 2024-11-24T22:16:23,116 DEBUG [RS_CLOSE_REGION-regionserver/bd6592c37bb2:0-0 {event_type=M_RS_CLOSE_REGION, pid=142}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/744bbec6934be304e01b009be760323d/.tmp/C/ab4e14cb47dd453a8f163f214c641153 as hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/744bbec6934be304e01b009be760323d/C/ab4e14cb47dd453a8f163f214c641153 2024-11-24T22:16:23,118 INFO [RS_CLOSE_REGION-regionserver/bd6592c37bb2:0-0 {event_type=M_RS_CLOSE_REGION, pid=142}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/744bbec6934be304e01b009be760323d/C/ab4e14cb47dd453a8f163f214c641153, entries=100, sequenceid=458, filesize=9.6 K 2024-11-24T22:16:23,119 INFO [RS_CLOSE_REGION-regionserver/bd6592c37bb2:0-0 {event_type=M_RS_CLOSE_REGION, pid=142}] regionserver.HRegion(3140): Finished flush of dataSize ~20.13 KB/20610, heapSize ~60.47 KB/61920, currentSize=0 B/0 for 744bbec6934be304e01b009be760323d in 1235ms, sequenceid=458, compaction requested=true 2024-11-24T22:16:23,120 DEBUG [StoreCloser-TestAcidGuarantees,,1732486556329.744bbec6934be304e01b009be760323d.-1 {}] regionserver.HStore(2317): Moving the files [hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/744bbec6934be304e01b009be760323d/A/56e7f0a11f784f479c172eeabafc39b7, hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/744bbec6934be304e01b009be760323d/A/9338a57d248a4d55bbcb179d0bffa465, hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/744bbec6934be304e01b009be760323d/A/87d7051b03c6450b9211666fb515ffe9, hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/744bbec6934be304e01b009be760323d/A/6a62440767db4e10be88de20dd521d76, hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/744bbec6934be304e01b009be760323d/A/1b438dc27ed04c2c9f0e4ec68f60b1e0, hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/744bbec6934be304e01b009be760323d/A/91027410d1ca4b5aa9256fa5d79892d5, hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/744bbec6934be304e01b009be760323d/A/85a7f1a085d7453a819dea01d918cb9a, hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/744bbec6934be304e01b009be760323d/A/b78790bd2d7849b18fe9b44f38b57ece, hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/744bbec6934be304e01b009be760323d/A/4678adece5b54edb9f8cb6392e04d599, hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/744bbec6934be304e01b009be760323d/A/488b263258da47168619dd5d260b7365, hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/744bbec6934be304e01b009be760323d/A/24a77bcdefc14b56820618f3519e5621, hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/744bbec6934be304e01b009be760323d/A/3691c3519de8465991e4b9e6f16216df, hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/744bbec6934be304e01b009be760323d/A/f98aceb87462436897276c8ce48119a3, hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/744bbec6934be304e01b009be760323d/A/9900ee0a105547bc9d0a8588ea156f77, hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/744bbec6934be304e01b009be760323d/A/2d64710994644bb48471f347abb9a937, hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/744bbec6934be304e01b009be760323d/A/845bfd56129a4b6fb3107b3ec0c44a38, hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/744bbec6934be304e01b009be760323d/A/f12797d1441c4e36baccfbffe71e8ad9, hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/744bbec6934be304e01b009be760323d/A/5ad98303c5714401b416c2ba4baaaff5, hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/744bbec6934be304e01b009be760323d/A/6b1fe8f1a1474845bad917d19afeb88c, hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/744bbec6934be304e01b009be760323d/A/021a9989574341f2a117574e6756a07a, hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/744bbec6934be304e01b009be760323d/A/5a670e57236e459fb71bd3c4bbdd6550, hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/744bbec6934be304e01b009be760323d/A/c6354d0aa5754431962b04fba90e09df, hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/744bbec6934be304e01b009be760323d/A/adba3952e07d44929aa9147ee59ac8fa, hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/744bbec6934be304e01b009be760323d/A/3bcf468781fb40189fb54b126de0bb40, hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/744bbec6934be304e01b009be760323d/A/4cbb2b50c7d74794986c166190e79fdb, hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/744bbec6934be304e01b009be760323d/A/8007d32c8dc5404ea79a5762cb35d83d, hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/744bbec6934be304e01b009be760323d/A/ed730b6e50aa4416aa0742d68965b490, hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/744bbec6934be304e01b009be760323d/A/513165e189654d61a14d6ba01582bce3, hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/744bbec6934be304e01b009be760323d/A/8939132dcea842a3bfda12ce12488284, hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/744bbec6934be304e01b009be760323d/A/bae78c8dcb824903b1bac1b8970098b3, hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/744bbec6934be304e01b009be760323d/A/ce29d063c8214791bbc3b99864b9d49e, hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/744bbec6934be304e01b009be760323d/A/f23753f45d0b43bbb035eefc6588d590] to archive 2024-11-24T22:16:23,120 DEBUG [StoreCloser-TestAcidGuarantees,,1732486556329.744bbec6934be304e01b009be760323d.-1 {}] backup.HFileArchiver(360): Archiving compacted files. 2024-11-24T22:16:23,121 DEBUG [StoreCloser-TestAcidGuarantees,,1732486556329.744bbec6934be304e01b009be760323d.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/744bbec6934be304e01b009be760323d/A/56e7f0a11f784f479c172eeabafc39b7 to hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/archive/data/default/TestAcidGuarantees/744bbec6934be304e01b009be760323d/A/56e7f0a11f784f479c172eeabafc39b7 2024-11-24T22:16:23,122 DEBUG [StoreCloser-TestAcidGuarantees,,1732486556329.744bbec6934be304e01b009be760323d.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/744bbec6934be304e01b009be760323d/A/9338a57d248a4d55bbcb179d0bffa465 to hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/archive/data/default/TestAcidGuarantees/744bbec6934be304e01b009be760323d/A/9338a57d248a4d55bbcb179d0bffa465 2024-11-24T22:16:23,123 DEBUG [StoreCloser-TestAcidGuarantees,,1732486556329.744bbec6934be304e01b009be760323d.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/744bbec6934be304e01b009be760323d/A/87d7051b03c6450b9211666fb515ffe9 to hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/archive/data/default/TestAcidGuarantees/744bbec6934be304e01b009be760323d/A/87d7051b03c6450b9211666fb515ffe9 2024-11-24T22:16:23,124 DEBUG [StoreCloser-TestAcidGuarantees,,1732486556329.744bbec6934be304e01b009be760323d.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/744bbec6934be304e01b009be760323d/A/6a62440767db4e10be88de20dd521d76 to hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/archive/data/default/TestAcidGuarantees/744bbec6934be304e01b009be760323d/A/6a62440767db4e10be88de20dd521d76 2024-11-24T22:16:23,125 DEBUG [StoreCloser-TestAcidGuarantees,,1732486556329.744bbec6934be304e01b009be760323d.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/744bbec6934be304e01b009be760323d/A/1b438dc27ed04c2c9f0e4ec68f60b1e0 to hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/archive/data/default/TestAcidGuarantees/744bbec6934be304e01b009be760323d/A/1b438dc27ed04c2c9f0e4ec68f60b1e0 2024-11-24T22:16:23,125 DEBUG [StoreCloser-TestAcidGuarantees,,1732486556329.744bbec6934be304e01b009be760323d.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/744bbec6934be304e01b009be760323d/A/91027410d1ca4b5aa9256fa5d79892d5 to hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/archive/data/default/TestAcidGuarantees/744bbec6934be304e01b009be760323d/A/91027410d1ca4b5aa9256fa5d79892d5 2024-11-24T22:16:23,126 DEBUG [StoreCloser-TestAcidGuarantees,,1732486556329.744bbec6934be304e01b009be760323d.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/744bbec6934be304e01b009be760323d/A/85a7f1a085d7453a819dea01d918cb9a to hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/archive/data/default/TestAcidGuarantees/744bbec6934be304e01b009be760323d/A/85a7f1a085d7453a819dea01d918cb9a 2024-11-24T22:16:23,127 DEBUG [StoreCloser-TestAcidGuarantees,,1732486556329.744bbec6934be304e01b009be760323d.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/744bbec6934be304e01b009be760323d/A/b78790bd2d7849b18fe9b44f38b57ece to hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/archive/data/default/TestAcidGuarantees/744bbec6934be304e01b009be760323d/A/b78790bd2d7849b18fe9b44f38b57ece 2024-11-24T22:16:23,128 DEBUG [StoreCloser-TestAcidGuarantees,,1732486556329.744bbec6934be304e01b009be760323d.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/744bbec6934be304e01b009be760323d/A/4678adece5b54edb9f8cb6392e04d599 to hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/archive/data/default/TestAcidGuarantees/744bbec6934be304e01b009be760323d/A/4678adece5b54edb9f8cb6392e04d599 2024-11-24T22:16:23,128 DEBUG [StoreCloser-TestAcidGuarantees,,1732486556329.744bbec6934be304e01b009be760323d.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/744bbec6934be304e01b009be760323d/A/488b263258da47168619dd5d260b7365 to hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/archive/data/default/TestAcidGuarantees/744bbec6934be304e01b009be760323d/A/488b263258da47168619dd5d260b7365 2024-11-24T22:16:23,129 DEBUG [StoreCloser-TestAcidGuarantees,,1732486556329.744bbec6934be304e01b009be760323d.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/744bbec6934be304e01b009be760323d/A/24a77bcdefc14b56820618f3519e5621 to hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/archive/data/default/TestAcidGuarantees/744bbec6934be304e01b009be760323d/A/24a77bcdefc14b56820618f3519e5621 2024-11-24T22:16:23,130 DEBUG [StoreCloser-TestAcidGuarantees,,1732486556329.744bbec6934be304e01b009be760323d.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/744bbec6934be304e01b009be760323d/A/3691c3519de8465991e4b9e6f16216df to hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/archive/data/default/TestAcidGuarantees/744bbec6934be304e01b009be760323d/A/3691c3519de8465991e4b9e6f16216df 2024-11-24T22:16:23,130 DEBUG [StoreCloser-TestAcidGuarantees,,1732486556329.744bbec6934be304e01b009be760323d.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/744bbec6934be304e01b009be760323d/A/f98aceb87462436897276c8ce48119a3 to hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/archive/data/default/TestAcidGuarantees/744bbec6934be304e01b009be760323d/A/f98aceb87462436897276c8ce48119a3 2024-11-24T22:16:23,131 DEBUG [StoreCloser-TestAcidGuarantees,,1732486556329.744bbec6934be304e01b009be760323d.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/744bbec6934be304e01b009be760323d/A/9900ee0a105547bc9d0a8588ea156f77 to hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/archive/data/default/TestAcidGuarantees/744bbec6934be304e01b009be760323d/A/9900ee0a105547bc9d0a8588ea156f77 2024-11-24T22:16:23,132 DEBUG [StoreCloser-TestAcidGuarantees,,1732486556329.744bbec6934be304e01b009be760323d.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/744bbec6934be304e01b009be760323d/A/2d64710994644bb48471f347abb9a937 to hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/archive/data/default/TestAcidGuarantees/744bbec6934be304e01b009be760323d/A/2d64710994644bb48471f347abb9a937 2024-11-24T22:16:23,132 DEBUG [StoreCloser-TestAcidGuarantees,,1732486556329.744bbec6934be304e01b009be760323d.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/744bbec6934be304e01b009be760323d/A/845bfd56129a4b6fb3107b3ec0c44a38 to hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/archive/data/default/TestAcidGuarantees/744bbec6934be304e01b009be760323d/A/845bfd56129a4b6fb3107b3ec0c44a38 2024-11-24T22:16:23,133 DEBUG [StoreCloser-TestAcidGuarantees,,1732486556329.744bbec6934be304e01b009be760323d.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/744bbec6934be304e01b009be760323d/A/f12797d1441c4e36baccfbffe71e8ad9 to hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/archive/data/default/TestAcidGuarantees/744bbec6934be304e01b009be760323d/A/f12797d1441c4e36baccfbffe71e8ad9 2024-11-24T22:16:23,134 DEBUG [StoreCloser-TestAcidGuarantees,,1732486556329.744bbec6934be304e01b009be760323d.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/744bbec6934be304e01b009be760323d/A/5ad98303c5714401b416c2ba4baaaff5 to hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/archive/data/default/TestAcidGuarantees/744bbec6934be304e01b009be760323d/A/5ad98303c5714401b416c2ba4baaaff5 2024-11-24T22:16:23,134 DEBUG [StoreCloser-TestAcidGuarantees,,1732486556329.744bbec6934be304e01b009be760323d.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/744bbec6934be304e01b009be760323d/A/6b1fe8f1a1474845bad917d19afeb88c to hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/archive/data/default/TestAcidGuarantees/744bbec6934be304e01b009be760323d/A/6b1fe8f1a1474845bad917d19afeb88c 2024-11-24T22:16:23,135 DEBUG [StoreCloser-TestAcidGuarantees,,1732486556329.744bbec6934be304e01b009be760323d.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/744bbec6934be304e01b009be760323d/A/021a9989574341f2a117574e6756a07a to hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/archive/data/default/TestAcidGuarantees/744bbec6934be304e01b009be760323d/A/021a9989574341f2a117574e6756a07a 2024-11-24T22:16:23,136 DEBUG [StoreCloser-TestAcidGuarantees,,1732486556329.744bbec6934be304e01b009be760323d.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/744bbec6934be304e01b009be760323d/A/5a670e57236e459fb71bd3c4bbdd6550 to hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/archive/data/default/TestAcidGuarantees/744bbec6934be304e01b009be760323d/A/5a670e57236e459fb71bd3c4bbdd6550 2024-11-24T22:16:23,136 DEBUG [StoreCloser-TestAcidGuarantees,,1732486556329.744bbec6934be304e01b009be760323d.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/744bbec6934be304e01b009be760323d/A/c6354d0aa5754431962b04fba90e09df to hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/archive/data/default/TestAcidGuarantees/744bbec6934be304e01b009be760323d/A/c6354d0aa5754431962b04fba90e09df 2024-11-24T22:16:23,137 DEBUG [StoreCloser-TestAcidGuarantees,,1732486556329.744bbec6934be304e01b009be760323d.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/744bbec6934be304e01b009be760323d/A/adba3952e07d44929aa9147ee59ac8fa to hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/archive/data/default/TestAcidGuarantees/744bbec6934be304e01b009be760323d/A/adba3952e07d44929aa9147ee59ac8fa 2024-11-24T22:16:23,138 DEBUG [StoreCloser-TestAcidGuarantees,,1732486556329.744bbec6934be304e01b009be760323d.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/744bbec6934be304e01b009be760323d/A/3bcf468781fb40189fb54b126de0bb40 to hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/archive/data/default/TestAcidGuarantees/744bbec6934be304e01b009be760323d/A/3bcf468781fb40189fb54b126de0bb40 2024-11-24T22:16:23,138 DEBUG [StoreCloser-TestAcidGuarantees,,1732486556329.744bbec6934be304e01b009be760323d.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/744bbec6934be304e01b009be760323d/A/4cbb2b50c7d74794986c166190e79fdb to hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/archive/data/default/TestAcidGuarantees/744bbec6934be304e01b009be760323d/A/4cbb2b50c7d74794986c166190e79fdb 2024-11-24T22:16:23,139 DEBUG [StoreCloser-TestAcidGuarantees,,1732486556329.744bbec6934be304e01b009be760323d.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/744bbec6934be304e01b009be760323d/A/8007d32c8dc5404ea79a5762cb35d83d to hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/archive/data/default/TestAcidGuarantees/744bbec6934be304e01b009be760323d/A/8007d32c8dc5404ea79a5762cb35d83d 2024-11-24T22:16:23,140 DEBUG [StoreCloser-TestAcidGuarantees,,1732486556329.744bbec6934be304e01b009be760323d.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/744bbec6934be304e01b009be760323d/A/ed730b6e50aa4416aa0742d68965b490 to hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/archive/data/default/TestAcidGuarantees/744bbec6934be304e01b009be760323d/A/ed730b6e50aa4416aa0742d68965b490 2024-11-24T22:16:23,140 DEBUG [StoreCloser-TestAcidGuarantees,,1732486556329.744bbec6934be304e01b009be760323d.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/744bbec6934be304e01b009be760323d/A/513165e189654d61a14d6ba01582bce3 to hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/archive/data/default/TestAcidGuarantees/744bbec6934be304e01b009be760323d/A/513165e189654d61a14d6ba01582bce3 2024-11-24T22:16:23,141 DEBUG [StoreCloser-TestAcidGuarantees,,1732486556329.744bbec6934be304e01b009be760323d.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/744bbec6934be304e01b009be760323d/A/8939132dcea842a3bfda12ce12488284 to hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/archive/data/default/TestAcidGuarantees/744bbec6934be304e01b009be760323d/A/8939132dcea842a3bfda12ce12488284 2024-11-24T22:16:23,142 DEBUG [StoreCloser-TestAcidGuarantees,,1732486556329.744bbec6934be304e01b009be760323d.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/744bbec6934be304e01b009be760323d/A/bae78c8dcb824903b1bac1b8970098b3 to hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/archive/data/default/TestAcidGuarantees/744bbec6934be304e01b009be760323d/A/bae78c8dcb824903b1bac1b8970098b3 2024-11-24T22:16:23,142 DEBUG [StoreCloser-TestAcidGuarantees,,1732486556329.744bbec6934be304e01b009be760323d.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/744bbec6934be304e01b009be760323d/A/ce29d063c8214791bbc3b99864b9d49e to hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/archive/data/default/TestAcidGuarantees/744bbec6934be304e01b009be760323d/A/ce29d063c8214791bbc3b99864b9d49e 2024-11-24T22:16:23,143 DEBUG [StoreCloser-TestAcidGuarantees,,1732486556329.744bbec6934be304e01b009be760323d.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/744bbec6934be304e01b009be760323d/A/f23753f45d0b43bbb035eefc6588d590 to hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/archive/data/default/TestAcidGuarantees/744bbec6934be304e01b009be760323d/A/f23753f45d0b43bbb035eefc6588d590 2024-11-24T22:16:23,144 DEBUG [StoreCloser-TestAcidGuarantees,,1732486556329.744bbec6934be304e01b009be760323d.-1 {}] regionserver.HStore(2317): Moving the files [hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/744bbec6934be304e01b009be760323d/B/6f1957bed3164d5aa2811fb5eb436e11, hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/744bbec6934be304e01b009be760323d/B/6d14ee73785241eca44e62a589b062d7, hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/744bbec6934be304e01b009be760323d/B/1a3fc19c62624ad0a67ac686e829085c, hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/744bbec6934be304e01b009be760323d/B/53ccdbef1ae447f1868f5cf9ab39d65e, hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/744bbec6934be304e01b009be760323d/B/884de7f2215840a584587eb367958865, hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/744bbec6934be304e01b009be760323d/B/9b45215aecb6437aa1a20c2a2e6823e5, hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/744bbec6934be304e01b009be760323d/B/2e3e28da1d0c43a1b0ce69ca0c9bebbb, hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/744bbec6934be304e01b009be760323d/B/04e097aec304409694fd69e1932b2a7b, hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/744bbec6934be304e01b009be760323d/B/d202f49d4dfd4adcacb0e187b0631626, hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/744bbec6934be304e01b009be760323d/B/39ecf63a40634ca0ac4b613e858c196c, hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/744bbec6934be304e01b009be760323d/B/ce895edc649449af9eabc9f2f406797a, hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/744bbec6934be304e01b009be760323d/B/4bf0ad08615e4d288f198e363bda6c8f, hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/744bbec6934be304e01b009be760323d/B/858191303eae4384ae680814f2697353, hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/744bbec6934be304e01b009be760323d/B/6cdd2aa199bc4abb9b0c181285f90fb1, hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/744bbec6934be304e01b009be760323d/B/2baf90c7fd214d82918f0a11a1398b68, hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/744bbec6934be304e01b009be760323d/B/015fadc872e249c49869f4546f14e186, hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/744bbec6934be304e01b009be760323d/B/dd7740f4d1404c9a8367cc54fe2fbdd3, hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/744bbec6934be304e01b009be760323d/B/bb1aba0dbb194f9c9e55d4e980089780, hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/744bbec6934be304e01b009be760323d/B/58027f2e64c3466da9d9ec87e36f74e6, hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/744bbec6934be304e01b009be760323d/B/0ee2bcf3005f4e8197e016dfb0809d3e, hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/744bbec6934be304e01b009be760323d/B/a4403f2f2fb04573be104f53fa890255, hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/744bbec6934be304e01b009be760323d/B/2f3eb3c994b34515add661fedd73ef59, hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/744bbec6934be304e01b009be760323d/B/7e1b795cbaa54f958590c9eee2e5bbf9, hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/744bbec6934be304e01b009be760323d/B/6ee3d0de54754f999dfa2ddfe20f7a94, hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/744bbec6934be304e01b009be760323d/B/3af7635b011644278e969391287c6f08, hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/744bbec6934be304e01b009be760323d/B/6fe913fa79a24f6fae7b349083870039, hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/744bbec6934be304e01b009be760323d/B/a25b2da6745546899c020b13033d9224, hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/744bbec6934be304e01b009be760323d/B/015f0d78f00e465198f4def8f41b689f, hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/744bbec6934be304e01b009be760323d/B/09209f77165d4e7f9f6c4c1ed40f9136, hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/744bbec6934be304e01b009be760323d/B/442e65fffb94469f8109f1c971c49198, hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/744bbec6934be304e01b009be760323d/B/b8f9a37fd50c47c0b695ccbd0b13417b, hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/744bbec6934be304e01b009be760323d/B/04d385c9cd224ba98ad89e3d72b81149] to archive 2024-11-24T22:16:23,145 DEBUG [StoreCloser-TestAcidGuarantees,,1732486556329.744bbec6934be304e01b009be760323d.-1 {}] backup.HFileArchiver(360): Archiving compacted files. 2024-11-24T22:16:23,146 DEBUG [StoreCloser-TestAcidGuarantees,,1732486556329.744bbec6934be304e01b009be760323d.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/744bbec6934be304e01b009be760323d/B/6f1957bed3164d5aa2811fb5eb436e11 to hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/archive/data/default/TestAcidGuarantees/744bbec6934be304e01b009be760323d/B/6f1957bed3164d5aa2811fb5eb436e11 2024-11-24T22:16:23,146 DEBUG [StoreCloser-TestAcidGuarantees,,1732486556329.744bbec6934be304e01b009be760323d.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/744bbec6934be304e01b009be760323d/B/6d14ee73785241eca44e62a589b062d7 to hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/archive/data/default/TestAcidGuarantees/744bbec6934be304e01b009be760323d/B/6d14ee73785241eca44e62a589b062d7 2024-11-24T22:16:23,147 DEBUG [StoreCloser-TestAcidGuarantees,,1732486556329.744bbec6934be304e01b009be760323d.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/744bbec6934be304e01b009be760323d/B/1a3fc19c62624ad0a67ac686e829085c to hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/archive/data/default/TestAcidGuarantees/744bbec6934be304e01b009be760323d/B/1a3fc19c62624ad0a67ac686e829085c 2024-11-24T22:16:23,148 DEBUG [StoreCloser-TestAcidGuarantees,,1732486556329.744bbec6934be304e01b009be760323d.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/744bbec6934be304e01b009be760323d/B/53ccdbef1ae447f1868f5cf9ab39d65e to hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/archive/data/default/TestAcidGuarantees/744bbec6934be304e01b009be760323d/B/53ccdbef1ae447f1868f5cf9ab39d65e 2024-11-24T22:16:23,148 DEBUG [StoreCloser-TestAcidGuarantees,,1732486556329.744bbec6934be304e01b009be760323d.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/744bbec6934be304e01b009be760323d/B/884de7f2215840a584587eb367958865 to hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/archive/data/default/TestAcidGuarantees/744bbec6934be304e01b009be760323d/B/884de7f2215840a584587eb367958865 2024-11-24T22:16:23,149 DEBUG [StoreCloser-TestAcidGuarantees,,1732486556329.744bbec6934be304e01b009be760323d.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/744bbec6934be304e01b009be760323d/B/9b45215aecb6437aa1a20c2a2e6823e5 to hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/archive/data/default/TestAcidGuarantees/744bbec6934be304e01b009be760323d/B/9b45215aecb6437aa1a20c2a2e6823e5 2024-11-24T22:16:23,150 DEBUG [StoreCloser-TestAcidGuarantees,,1732486556329.744bbec6934be304e01b009be760323d.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/744bbec6934be304e01b009be760323d/B/2e3e28da1d0c43a1b0ce69ca0c9bebbb to hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/archive/data/default/TestAcidGuarantees/744bbec6934be304e01b009be760323d/B/2e3e28da1d0c43a1b0ce69ca0c9bebbb 2024-11-24T22:16:23,151 DEBUG [StoreCloser-TestAcidGuarantees,,1732486556329.744bbec6934be304e01b009be760323d.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/744bbec6934be304e01b009be760323d/B/04e097aec304409694fd69e1932b2a7b to hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/archive/data/default/TestAcidGuarantees/744bbec6934be304e01b009be760323d/B/04e097aec304409694fd69e1932b2a7b 2024-11-24T22:16:23,151 DEBUG [StoreCloser-TestAcidGuarantees,,1732486556329.744bbec6934be304e01b009be760323d.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/744bbec6934be304e01b009be760323d/B/d202f49d4dfd4adcacb0e187b0631626 to hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/archive/data/default/TestAcidGuarantees/744bbec6934be304e01b009be760323d/B/d202f49d4dfd4adcacb0e187b0631626 2024-11-24T22:16:23,152 DEBUG [StoreCloser-TestAcidGuarantees,,1732486556329.744bbec6934be304e01b009be760323d.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/744bbec6934be304e01b009be760323d/B/39ecf63a40634ca0ac4b613e858c196c to hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/archive/data/default/TestAcidGuarantees/744bbec6934be304e01b009be760323d/B/39ecf63a40634ca0ac4b613e858c196c 2024-11-24T22:16:23,153 DEBUG [StoreCloser-TestAcidGuarantees,,1732486556329.744bbec6934be304e01b009be760323d.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/744bbec6934be304e01b009be760323d/B/ce895edc649449af9eabc9f2f406797a to hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/archive/data/default/TestAcidGuarantees/744bbec6934be304e01b009be760323d/B/ce895edc649449af9eabc9f2f406797a 2024-11-24T22:16:23,153 DEBUG [StoreCloser-TestAcidGuarantees,,1732486556329.744bbec6934be304e01b009be760323d.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/744bbec6934be304e01b009be760323d/B/4bf0ad08615e4d288f198e363bda6c8f to hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/archive/data/default/TestAcidGuarantees/744bbec6934be304e01b009be760323d/B/4bf0ad08615e4d288f198e363bda6c8f 2024-11-24T22:16:23,154 DEBUG [StoreCloser-TestAcidGuarantees,,1732486556329.744bbec6934be304e01b009be760323d.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/744bbec6934be304e01b009be760323d/B/858191303eae4384ae680814f2697353 to hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/archive/data/default/TestAcidGuarantees/744bbec6934be304e01b009be760323d/B/858191303eae4384ae680814f2697353 2024-11-24T22:16:23,155 DEBUG [StoreCloser-TestAcidGuarantees,,1732486556329.744bbec6934be304e01b009be760323d.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/744bbec6934be304e01b009be760323d/B/6cdd2aa199bc4abb9b0c181285f90fb1 to hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/archive/data/default/TestAcidGuarantees/744bbec6934be304e01b009be760323d/B/6cdd2aa199bc4abb9b0c181285f90fb1 2024-11-24T22:16:23,156 DEBUG [StoreCloser-TestAcidGuarantees,,1732486556329.744bbec6934be304e01b009be760323d.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/744bbec6934be304e01b009be760323d/B/2baf90c7fd214d82918f0a11a1398b68 to hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/archive/data/default/TestAcidGuarantees/744bbec6934be304e01b009be760323d/B/2baf90c7fd214d82918f0a11a1398b68 2024-11-24T22:16:23,156 DEBUG [StoreCloser-TestAcidGuarantees,,1732486556329.744bbec6934be304e01b009be760323d.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/744bbec6934be304e01b009be760323d/B/015fadc872e249c49869f4546f14e186 to hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/archive/data/default/TestAcidGuarantees/744bbec6934be304e01b009be760323d/B/015fadc872e249c49869f4546f14e186 2024-11-24T22:16:23,157 DEBUG [StoreCloser-TestAcidGuarantees,,1732486556329.744bbec6934be304e01b009be760323d.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/744bbec6934be304e01b009be760323d/B/dd7740f4d1404c9a8367cc54fe2fbdd3 to hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/archive/data/default/TestAcidGuarantees/744bbec6934be304e01b009be760323d/B/dd7740f4d1404c9a8367cc54fe2fbdd3 2024-11-24T22:16:23,158 DEBUG [StoreCloser-TestAcidGuarantees,,1732486556329.744bbec6934be304e01b009be760323d.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/744bbec6934be304e01b009be760323d/B/bb1aba0dbb194f9c9e55d4e980089780 to hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/archive/data/default/TestAcidGuarantees/744bbec6934be304e01b009be760323d/B/bb1aba0dbb194f9c9e55d4e980089780 2024-11-24T22:16:23,158 DEBUG [StoreCloser-TestAcidGuarantees,,1732486556329.744bbec6934be304e01b009be760323d.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/744bbec6934be304e01b009be760323d/B/58027f2e64c3466da9d9ec87e36f74e6 to hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/archive/data/default/TestAcidGuarantees/744bbec6934be304e01b009be760323d/B/58027f2e64c3466da9d9ec87e36f74e6 2024-11-24T22:16:23,159 DEBUG [StoreCloser-TestAcidGuarantees,,1732486556329.744bbec6934be304e01b009be760323d.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/744bbec6934be304e01b009be760323d/B/0ee2bcf3005f4e8197e016dfb0809d3e to hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/archive/data/default/TestAcidGuarantees/744bbec6934be304e01b009be760323d/B/0ee2bcf3005f4e8197e016dfb0809d3e 2024-11-24T22:16:23,160 DEBUG [StoreCloser-TestAcidGuarantees,,1732486556329.744bbec6934be304e01b009be760323d.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/744bbec6934be304e01b009be760323d/B/a4403f2f2fb04573be104f53fa890255 to hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/archive/data/default/TestAcidGuarantees/744bbec6934be304e01b009be760323d/B/a4403f2f2fb04573be104f53fa890255 2024-11-24T22:16:23,161 DEBUG [StoreCloser-TestAcidGuarantees,,1732486556329.744bbec6934be304e01b009be760323d.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/744bbec6934be304e01b009be760323d/B/2f3eb3c994b34515add661fedd73ef59 to hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/archive/data/default/TestAcidGuarantees/744bbec6934be304e01b009be760323d/B/2f3eb3c994b34515add661fedd73ef59 2024-11-24T22:16:23,161 DEBUG [StoreCloser-TestAcidGuarantees,,1732486556329.744bbec6934be304e01b009be760323d.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/744bbec6934be304e01b009be760323d/B/7e1b795cbaa54f958590c9eee2e5bbf9 to hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/archive/data/default/TestAcidGuarantees/744bbec6934be304e01b009be760323d/B/7e1b795cbaa54f958590c9eee2e5bbf9 2024-11-24T22:16:23,162 DEBUG [StoreCloser-TestAcidGuarantees,,1732486556329.744bbec6934be304e01b009be760323d.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/744bbec6934be304e01b009be760323d/B/6ee3d0de54754f999dfa2ddfe20f7a94 to hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/archive/data/default/TestAcidGuarantees/744bbec6934be304e01b009be760323d/B/6ee3d0de54754f999dfa2ddfe20f7a94 2024-11-24T22:16:23,163 DEBUG [StoreCloser-TestAcidGuarantees,,1732486556329.744bbec6934be304e01b009be760323d.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/744bbec6934be304e01b009be760323d/B/3af7635b011644278e969391287c6f08 to hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/archive/data/default/TestAcidGuarantees/744bbec6934be304e01b009be760323d/B/3af7635b011644278e969391287c6f08 2024-11-24T22:16:23,164 DEBUG [StoreCloser-TestAcidGuarantees,,1732486556329.744bbec6934be304e01b009be760323d.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/744bbec6934be304e01b009be760323d/B/6fe913fa79a24f6fae7b349083870039 to hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/archive/data/default/TestAcidGuarantees/744bbec6934be304e01b009be760323d/B/6fe913fa79a24f6fae7b349083870039 2024-11-24T22:16:23,164 DEBUG [StoreCloser-TestAcidGuarantees,,1732486556329.744bbec6934be304e01b009be760323d.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/744bbec6934be304e01b009be760323d/B/a25b2da6745546899c020b13033d9224 to hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/archive/data/default/TestAcidGuarantees/744bbec6934be304e01b009be760323d/B/a25b2da6745546899c020b13033d9224 2024-11-24T22:16:23,165 DEBUG [StoreCloser-TestAcidGuarantees,,1732486556329.744bbec6934be304e01b009be760323d.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/744bbec6934be304e01b009be760323d/B/015f0d78f00e465198f4def8f41b689f to hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/archive/data/default/TestAcidGuarantees/744bbec6934be304e01b009be760323d/B/015f0d78f00e465198f4def8f41b689f 2024-11-24T22:16:23,166 DEBUG [StoreCloser-TestAcidGuarantees,,1732486556329.744bbec6934be304e01b009be760323d.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/744bbec6934be304e01b009be760323d/B/09209f77165d4e7f9f6c4c1ed40f9136 to hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/archive/data/default/TestAcidGuarantees/744bbec6934be304e01b009be760323d/B/09209f77165d4e7f9f6c4c1ed40f9136 2024-11-24T22:16:23,166 DEBUG [StoreCloser-TestAcidGuarantees,,1732486556329.744bbec6934be304e01b009be760323d.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/744bbec6934be304e01b009be760323d/B/442e65fffb94469f8109f1c971c49198 to hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/archive/data/default/TestAcidGuarantees/744bbec6934be304e01b009be760323d/B/442e65fffb94469f8109f1c971c49198 2024-11-24T22:16:23,167 DEBUG [StoreCloser-TestAcidGuarantees,,1732486556329.744bbec6934be304e01b009be760323d.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/744bbec6934be304e01b009be760323d/B/b8f9a37fd50c47c0b695ccbd0b13417b to hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/archive/data/default/TestAcidGuarantees/744bbec6934be304e01b009be760323d/B/b8f9a37fd50c47c0b695ccbd0b13417b 2024-11-24T22:16:23,168 DEBUG [StoreCloser-TestAcidGuarantees,,1732486556329.744bbec6934be304e01b009be760323d.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/744bbec6934be304e01b009be760323d/B/04d385c9cd224ba98ad89e3d72b81149 to hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/archive/data/default/TestAcidGuarantees/744bbec6934be304e01b009be760323d/B/04d385c9cd224ba98ad89e3d72b81149 2024-11-24T22:16:23,169 DEBUG [StoreCloser-TestAcidGuarantees,,1732486556329.744bbec6934be304e01b009be760323d.-1 {}] regionserver.HStore(2317): Moving the files [hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/744bbec6934be304e01b009be760323d/C/d9a1c99c3bfb401092039d1293ad1252, hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/744bbec6934be304e01b009be760323d/C/70cb46b606d741fb9cfa26d290b0aac8, hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/744bbec6934be304e01b009be760323d/C/57a5000178934a4c98dbc71d7e5ffb12, hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/744bbec6934be304e01b009be760323d/C/33440ac6cc804bad90dd4f9ef9db6daa, hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/744bbec6934be304e01b009be760323d/C/04d402c6aca248ec96f071d468ea7602, hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/744bbec6934be304e01b009be760323d/C/dfd78c5f1e3446ceb9e6877167ddb6c0, hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/744bbec6934be304e01b009be760323d/C/a21678a1374048da9ca65db0afd717dc, hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/744bbec6934be304e01b009be760323d/C/a7b5aae7560f43f3af42b146d01b7d87, hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/744bbec6934be304e01b009be760323d/C/0ad322a0749345fdb793029953abefd7, hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/744bbec6934be304e01b009be760323d/C/98ae469563954c268cb17c800408d0ea, hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/744bbec6934be304e01b009be760323d/C/e04a440a3f514792b596de994ba37e38, hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/744bbec6934be304e01b009be760323d/C/958ad208c5ba4d749d3f1c406bdcd7f7, hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/744bbec6934be304e01b009be760323d/C/d53b3ca00fc140c6b59fdb5c520720c7, hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/744bbec6934be304e01b009be760323d/C/e45e5aa1f5c44dabadda8d63a9a13c5d, hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/744bbec6934be304e01b009be760323d/C/73124dd39e54419f99732bf24548a36d, hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/744bbec6934be304e01b009be760323d/C/ea607d43102d4e1398667662a8d59ffe, hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/744bbec6934be304e01b009be760323d/C/fd0b85f893e14463833221db0d0fcbf9, hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/744bbec6934be304e01b009be760323d/C/21731928879347bfbe3d06c90347654e, hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/744bbec6934be304e01b009be760323d/C/90db8ce5d3d54446be80c89ee834828f, hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/744bbec6934be304e01b009be760323d/C/76252ddb4dd84d358abcfb118a4bb6b4, hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/744bbec6934be304e01b009be760323d/C/3516e75f79734fb39d6757fe2a86a014, hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/744bbec6934be304e01b009be760323d/C/2583f4800d584c48b0cf73a317359c00, hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/744bbec6934be304e01b009be760323d/C/3e9d21519ab34af298726d4eedaf4608, hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/744bbec6934be304e01b009be760323d/C/18cfa9a432cd4a559609140d0121d730, hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/744bbec6934be304e01b009be760323d/C/5325c2d24c764c4eb297525675256fbd, hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/744bbec6934be304e01b009be760323d/C/5d83f39257e349a0b43d8f3dd1fdf19b, hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/744bbec6934be304e01b009be760323d/C/a23bbfecd00441eab0a749d4ed2a5cd5, hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/744bbec6934be304e01b009be760323d/C/05ec45638c4643b1b3313cdf31e2f636, hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/744bbec6934be304e01b009be760323d/C/d2585fa8933643ddb5981789762dddba, hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/744bbec6934be304e01b009be760323d/C/f67ba0237d4c48d8b9ce5f846119b2ba, hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/744bbec6934be304e01b009be760323d/C/7b7ae2272b0f466688e0574d67613286, hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/744bbec6934be304e01b009be760323d/C/929ecd28586e42e1ab4a45d46f0de599] to archive 2024-11-24T22:16:23,170 DEBUG [StoreCloser-TestAcidGuarantees,,1732486556329.744bbec6934be304e01b009be760323d.-1 {}] backup.HFileArchiver(360): Archiving compacted files. 2024-11-24T22:16:23,171 DEBUG [StoreCloser-TestAcidGuarantees,,1732486556329.744bbec6934be304e01b009be760323d.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/744bbec6934be304e01b009be760323d/C/d9a1c99c3bfb401092039d1293ad1252 to hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/archive/data/default/TestAcidGuarantees/744bbec6934be304e01b009be760323d/C/d9a1c99c3bfb401092039d1293ad1252 2024-11-24T22:16:23,172 DEBUG [StoreCloser-TestAcidGuarantees,,1732486556329.744bbec6934be304e01b009be760323d.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/744bbec6934be304e01b009be760323d/C/70cb46b606d741fb9cfa26d290b0aac8 to hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/archive/data/default/TestAcidGuarantees/744bbec6934be304e01b009be760323d/C/70cb46b606d741fb9cfa26d290b0aac8 2024-11-24T22:16:23,173 DEBUG [StoreCloser-TestAcidGuarantees,,1732486556329.744bbec6934be304e01b009be760323d.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/744bbec6934be304e01b009be760323d/C/57a5000178934a4c98dbc71d7e5ffb12 to hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/archive/data/default/TestAcidGuarantees/744bbec6934be304e01b009be760323d/C/57a5000178934a4c98dbc71d7e5ffb12 2024-11-24T22:16:23,174 DEBUG [StoreCloser-TestAcidGuarantees,,1732486556329.744bbec6934be304e01b009be760323d.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/744bbec6934be304e01b009be760323d/C/33440ac6cc804bad90dd4f9ef9db6daa to hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/archive/data/default/TestAcidGuarantees/744bbec6934be304e01b009be760323d/C/33440ac6cc804bad90dd4f9ef9db6daa 2024-11-24T22:16:23,174 DEBUG [StoreCloser-TestAcidGuarantees,,1732486556329.744bbec6934be304e01b009be760323d.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/744bbec6934be304e01b009be760323d/C/04d402c6aca248ec96f071d468ea7602 to hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/archive/data/default/TestAcidGuarantees/744bbec6934be304e01b009be760323d/C/04d402c6aca248ec96f071d468ea7602 2024-11-24T22:16:23,175 DEBUG [StoreCloser-TestAcidGuarantees,,1732486556329.744bbec6934be304e01b009be760323d.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/744bbec6934be304e01b009be760323d/C/dfd78c5f1e3446ceb9e6877167ddb6c0 to hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/archive/data/default/TestAcidGuarantees/744bbec6934be304e01b009be760323d/C/dfd78c5f1e3446ceb9e6877167ddb6c0 2024-11-24T22:16:23,176 DEBUG [StoreCloser-TestAcidGuarantees,,1732486556329.744bbec6934be304e01b009be760323d.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/744bbec6934be304e01b009be760323d/C/a21678a1374048da9ca65db0afd717dc to hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/archive/data/default/TestAcidGuarantees/744bbec6934be304e01b009be760323d/C/a21678a1374048da9ca65db0afd717dc 2024-11-24T22:16:23,177 DEBUG [StoreCloser-TestAcidGuarantees,,1732486556329.744bbec6934be304e01b009be760323d.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/744bbec6934be304e01b009be760323d/C/a7b5aae7560f43f3af42b146d01b7d87 to hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/archive/data/default/TestAcidGuarantees/744bbec6934be304e01b009be760323d/C/a7b5aae7560f43f3af42b146d01b7d87 2024-11-24T22:16:23,178 DEBUG [StoreCloser-TestAcidGuarantees,,1732486556329.744bbec6934be304e01b009be760323d.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/744bbec6934be304e01b009be760323d/C/0ad322a0749345fdb793029953abefd7 to hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/archive/data/default/TestAcidGuarantees/744bbec6934be304e01b009be760323d/C/0ad322a0749345fdb793029953abefd7 2024-11-24T22:16:23,178 DEBUG [StoreCloser-TestAcidGuarantees,,1732486556329.744bbec6934be304e01b009be760323d.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/744bbec6934be304e01b009be760323d/C/98ae469563954c268cb17c800408d0ea to hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/archive/data/default/TestAcidGuarantees/744bbec6934be304e01b009be760323d/C/98ae469563954c268cb17c800408d0ea 2024-11-24T22:16:23,179 DEBUG [StoreCloser-TestAcidGuarantees,,1732486556329.744bbec6934be304e01b009be760323d.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/744bbec6934be304e01b009be760323d/C/e04a440a3f514792b596de994ba37e38 to hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/archive/data/default/TestAcidGuarantees/744bbec6934be304e01b009be760323d/C/e04a440a3f514792b596de994ba37e38 2024-11-24T22:16:23,180 DEBUG [StoreCloser-TestAcidGuarantees,,1732486556329.744bbec6934be304e01b009be760323d.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/744bbec6934be304e01b009be760323d/C/958ad208c5ba4d749d3f1c406bdcd7f7 to hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/archive/data/default/TestAcidGuarantees/744bbec6934be304e01b009be760323d/C/958ad208c5ba4d749d3f1c406bdcd7f7 2024-11-24T22:16:23,180 DEBUG [StoreCloser-TestAcidGuarantees,,1732486556329.744bbec6934be304e01b009be760323d.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/744bbec6934be304e01b009be760323d/C/d53b3ca00fc140c6b59fdb5c520720c7 to hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/archive/data/default/TestAcidGuarantees/744bbec6934be304e01b009be760323d/C/d53b3ca00fc140c6b59fdb5c520720c7 2024-11-24T22:16:23,181 DEBUG [StoreCloser-TestAcidGuarantees,,1732486556329.744bbec6934be304e01b009be760323d.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/744bbec6934be304e01b009be760323d/C/e45e5aa1f5c44dabadda8d63a9a13c5d to hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/archive/data/default/TestAcidGuarantees/744bbec6934be304e01b009be760323d/C/e45e5aa1f5c44dabadda8d63a9a13c5d 2024-11-24T22:16:23,182 DEBUG [StoreCloser-TestAcidGuarantees,,1732486556329.744bbec6934be304e01b009be760323d.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/744bbec6934be304e01b009be760323d/C/73124dd39e54419f99732bf24548a36d to hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/archive/data/default/TestAcidGuarantees/744bbec6934be304e01b009be760323d/C/73124dd39e54419f99732bf24548a36d 2024-11-24T22:16:23,183 DEBUG [StoreCloser-TestAcidGuarantees,,1732486556329.744bbec6934be304e01b009be760323d.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/744bbec6934be304e01b009be760323d/C/ea607d43102d4e1398667662a8d59ffe to hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/archive/data/default/TestAcidGuarantees/744bbec6934be304e01b009be760323d/C/ea607d43102d4e1398667662a8d59ffe 2024-11-24T22:16:23,183 DEBUG [StoreCloser-TestAcidGuarantees,,1732486556329.744bbec6934be304e01b009be760323d.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/744bbec6934be304e01b009be760323d/C/fd0b85f893e14463833221db0d0fcbf9 to hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/archive/data/default/TestAcidGuarantees/744bbec6934be304e01b009be760323d/C/fd0b85f893e14463833221db0d0fcbf9 2024-11-24T22:16:23,184 DEBUG [StoreCloser-TestAcidGuarantees,,1732486556329.744bbec6934be304e01b009be760323d.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/744bbec6934be304e01b009be760323d/C/21731928879347bfbe3d06c90347654e to hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/archive/data/default/TestAcidGuarantees/744bbec6934be304e01b009be760323d/C/21731928879347bfbe3d06c90347654e 2024-11-24T22:16:23,185 DEBUG [StoreCloser-TestAcidGuarantees,,1732486556329.744bbec6934be304e01b009be760323d.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/744bbec6934be304e01b009be760323d/C/90db8ce5d3d54446be80c89ee834828f to hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/archive/data/default/TestAcidGuarantees/744bbec6934be304e01b009be760323d/C/90db8ce5d3d54446be80c89ee834828f 2024-11-24T22:16:23,185 DEBUG [StoreCloser-TestAcidGuarantees,,1732486556329.744bbec6934be304e01b009be760323d.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/744bbec6934be304e01b009be760323d/C/76252ddb4dd84d358abcfb118a4bb6b4 to hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/archive/data/default/TestAcidGuarantees/744bbec6934be304e01b009be760323d/C/76252ddb4dd84d358abcfb118a4bb6b4 2024-11-24T22:16:23,186 DEBUG [StoreCloser-TestAcidGuarantees,,1732486556329.744bbec6934be304e01b009be760323d.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/744bbec6934be304e01b009be760323d/C/3516e75f79734fb39d6757fe2a86a014 to hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/archive/data/default/TestAcidGuarantees/744bbec6934be304e01b009be760323d/C/3516e75f79734fb39d6757fe2a86a014 2024-11-24T22:16:23,187 DEBUG [StoreCloser-TestAcidGuarantees,,1732486556329.744bbec6934be304e01b009be760323d.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/744bbec6934be304e01b009be760323d/C/2583f4800d584c48b0cf73a317359c00 to hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/archive/data/default/TestAcidGuarantees/744bbec6934be304e01b009be760323d/C/2583f4800d584c48b0cf73a317359c00 2024-11-24T22:16:23,187 DEBUG [StoreCloser-TestAcidGuarantees,,1732486556329.744bbec6934be304e01b009be760323d.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/744bbec6934be304e01b009be760323d/C/3e9d21519ab34af298726d4eedaf4608 to hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/archive/data/default/TestAcidGuarantees/744bbec6934be304e01b009be760323d/C/3e9d21519ab34af298726d4eedaf4608 2024-11-24T22:16:23,188 DEBUG [StoreCloser-TestAcidGuarantees,,1732486556329.744bbec6934be304e01b009be760323d.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/744bbec6934be304e01b009be760323d/C/18cfa9a432cd4a559609140d0121d730 to hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/archive/data/default/TestAcidGuarantees/744bbec6934be304e01b009be760323d/C/18cfa9a432cd4a559609140d0121d730 2024-11-24T22:16:23,189 DEBUG [StoreCloser-TestAcidGuarantees,,1732486556329.744bbec6934be304e01b009be760323d.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/744bbec6934be304e01b009be760323d/C/5325c2d24c764c4eb297525675256fbd to hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/archive/data/default/TestAcidGuarantees/744bbec6934be304e01b009be760323d/C/5325c2d24c764c4eb297525675256fbd 2024-11-24T22:16:23,189 DEBUG [StoreCloser-TestAcidGuarantees,,1732486556329.744bbec6934be304e01b009be760323d.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/744bbec6934be304e01b009be760323d/C/5d83f39257e349a0b43d8f3dd1fdf19b to hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/archive/data/default/TestAcidGuarantees/744bbec6934be304e01b009be760323d/C/5d83f39257e349a0b43d8f3dd1fdf19b 2024-11-24T22:16:23,190 DEBUG [StoreCloser-TestAcidGuarantees,,1732486556329.744bbec6934be304e01b009be760323d.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/744bbec6934be304e01b009be760323d/C/a23bbfecd00441eab0a749d4ed2a5cd5 to hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/archive/data/default/TestAcidGuarantees/744bbec6934be304e01b009be760323d/C/a23bbfecd00441eab0a749d4ed2a5cd5 2024-11-24T22:16:23,191 DEBUG [StoreCloser-TestAcidGuarantees,,1732486556329.744bbec6934be304e01b009be760323d.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/744bbec6934be304e01b009be760323d/C/05ec45638c4643b1b3313cdf31e2f636 to hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/archive/data/default/TestAcidGuarantees/744bbec6934be304e01b009be760323d/C/05ec45638c4643b1b3313cdf31e2f636 2024-11-24T22:16:23,192 DEBUG [StoreCloser-TestAcidGuarantees,,1732486556329.744bbec6934be304e01b009be760323d.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/744bbec6934be304e01b009be760323d/C/d2585fa8933643ddb5981789762dddba to hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/archive/data/default/TestAcidGuarantees/744bbec6934be304e01b009be760323d/C/d2585fa8933643ddb5981789762dddba 2024-11-24T22:16:23,193 DEBUG [StoreCloser-TestAcidGuarantees,,1732486556329.744bbec6934be304e01b009be760323d.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/744bbec6934be304e01b009be760323d/C/f67ba0237d4c48d8b9ce5f846119b2ba to hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/archive/data/default/TestAcidGuarantees/744bbec6934be304e01b009be760323d/C/f67ba0237d4c48d8b9ce5f846119b2ba 2024-11-24T22:16:23,194 DEBUG [StoreCloser-TestAcidGuarantees,,1732486556329.744bbec6934be304e01b009be760323d.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/744bbec6934be304e01b009be760323d/C/7b7ae2272b0f466688e0574d67613286 to hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/archive/data/default/TestAcidGuarantees/744bbec6934be304e01b009be760323d/C/7b7ae2272b0f466688e0574d67613286 2024-11-24T22:16:23,195 DEBUG [StoreCloser-TestAcidGuarantees,,1732486556329.744bbec6934be304e01b009be760323d.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/744bbec6934be304e01b009be760323d/C/929ecd28586e42e1ab4a45d46f0de599 to hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/archive/data/default/TestAcidGuarantees/744bbec6934be304e01b009be760323d/C/929ecd28586e42e1ab4a45d46f0de599 2024-11-24T22:16:23,198 DEBUG [RS_CLOSE_REGION-regionserver/bd6592c37bb2:0-0 {event_type=M_RS_CLOSE_REGION, pid=142}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/744bbec6934be304e01b009be760323d/recovered.edits/461.seqid, newMaxSeqId=461, maxSeqId=1 2024-11-24T22:16:23,198 INFO [RS_CLOSE_REGION-regionserver/bd6592c37bb2:0-0 {event_type=M_RS_CLOSE_REGION, pid=142}] regionserver.HRegion(1973): Closed TestAcidGuarantees,,1732486556329.744bbec6934be304e01b009be760323d. 2024-11-24T22:16:23,198 DEBUG [RS_CLOSE_REGION-regionserver/bd6592c37bb2:0-0 {event_type=M_RS_CLOSE_REGION, pid=142}] regionserver.HRegion(1676): Region close journal for 744bbec6934be304e01b009be760323d: Waiting for close lock at 1732486581213Running coprocessor pre-close hooks at 1732486581213Disabling compacts and flushes for region at 1732486581213Disabling writes for close at 1732486581884 (+671 ms)Obtaining lock to block concurrent updates at 1732486581884Preparing flush snapshotting stores in 744bbec6934be304e01b009be760323d at 1732486581884Finished memstore snapshotting TestAcidGuarantees,,1732486556329.744bbec6934be304e01b009be760323d., syncing WAL and waiting on mvcc, flushsize=dataSize=20610, getHeapSize=61920, getOffHeapSize=0, getCellsCount=450 at 1732486581884Flushing stores of TestAcidGuarantees,,1732486556329.744bbec6934be304e01b009be760323d. at 1732486581885 (+1 ms)Flushing 744bbec6934be304e01b009be760323d/A: creating writer at 1732486581885Flushing 744bbec6934be304e01b009be760323d/A: appending metadata at 1732486581887 (+2 ms)Flushing 744bbec6934be304e01b009be760323d/A: closing flushed file at 1732486581887Flushing 744bbec6934be304e01b009be760323d/B: creating writer at 1732486582293 (+406 ms)Flushing 744bbec6934be304e01b009be760323d/B: appending metadata at 1732486582295 (+2 ms)Flushing 744bbec6934be304e01b009be760323d/B: closing flushed file at 1732486582295Flushing 744bbec6934be304e01b009be760323d/C: creating writer at 1732486582702 (+407 ms)Flushing 744bbec6934be304e01b009be760323d/C: appending metadata at 1732486582704 (+2 ms)Flushing 744bbec6934be304e01b009be760323d/C: closing flushed file at 1732486582704Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@7fa0f80b: reopening flushed file at 1732486583110 (+406 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@3c1ef8d6: reopening flushed file at 1732486583113 (+3 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@359f37c3: reopening flushed file at 1732486583116 (+3 ms)Finished flush of dataSize ~20.13 KB/20610, heapSize ~60.47 KB/61920, currentSize=0 B/0 for 744bbec6934be304e01b009be760323d in 1235ms, sequenceid=458, compaction requested=true at 1732486583119 (+3 ms)Writing region close event to WAL at 1732486583196 (+77 ms)Running coprocessor post-close hooks at 1732486583198 (+2 ms)Closed at 1732486583198 2024-11-24T22:16:23,200 INFO [RS_CLOSE_REGION-regionserver/bd6592c37bb2:0-0 {event_type=M_RS_CLOSE_REGION, pid=142}] handler.UnassignRegionHandler(157): Closed 744bbec6934be304e01b009be760323d 2024-11-24T22:16:23,200 INFO [PEWorker-3 {}] assignment.RegionStateStore(223): pid=141 updating hbase:meta row=744bbec6934be304e01b009be760323d, regionState=CLOSED 2024-11-24T22:16:23,201 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37701 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=139 2024-11-24T22:16:23,201 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=142, ppid=141, state=RUNNABLE, hasLock=false; CloseRegionProcedure 744bbec6934be304e01b009be760323d, server=bd6592c37bb2,40087,1732486423713 because future has completed 2024-11-24T22:16:23,203 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=142, resume processing ppid=141 2024-11-24T22:16:23,203 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=142, ppid=141, state=SUCCESS, hasLock=false; CloseRegionProcedure 744bbec6934be304e01b009be760323d, server=bd6592c37bb2,40087,1732486423713 in 2.1410 sec 2024-11-24T22:16:23,204 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=141, resume processing ppid=140 2024-11-24T22:16:23,204 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=141, ppid=140, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=744bbec6934be304e01b009be760323d, UNASSIGN in 2.1440 sec 2024-11-24T22:16:23,206 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=140, resume processing ppid=139 2024-11-24T22:16:23,206 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=140, ppid=139, state=SUCCESS, hasLock=false; CloseTableRegionsProcedure table=TestAcidGuarantees in 2.1470 sec 2024-11-24T22:16:23,207 DEBUG [PEWorker-4 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1732486583207"}]},"ts":"1732486583207"} 2024-11-24T22:16:23,208 INFO [PEWorker-4 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=DISABLED in hbase:meta 2024-11-24T22:16:23,208 INFO [PEWorker-4 {}] procedure.DisableTableProcedure(296): Set TestAcidGuarantees to state=DISABLED 2024-11-24T22:16:23,210 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=139, state=SUCCESS, hasLock=false; DisableTableProcedure table=TestAcidGuarantees in 2.1550 sec 2024-11-24T22:16:23,413 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(153): Removing adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_default_table_TestAcidGuarantees 2024-11-24T22:16:25,211 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37701 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=139 2024-11-24T22:16:25,211 INFO [RPCClient-NioEventLoopGroup-4-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: DISABLE, Table Name: default:TestAcidGuarantees completed 2024-11-24T22:16:25,211 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37701 {}] master.HMaster$5(2570): Client=jenkins//172.17.0.2 delete TestAcidGuarantees 2024-11-24T22:16:25,212 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37701 {}] procedure2.ProcedureExecutor(1139): Stored pid=143, state=RUNNABLE:DELETE_TABLE_PRE_OPERATION, hasLock=false; DeleteTableProcedure table=TestAcidGuarantees 2024-11-24T22:16:25,213 DEBUG [PEWorker-3 {}] procedure.DeleteTableProcedure(103): Waiting for RIT for pid=143, state=RUNNABLE:DELETE_TABLE_PRE_OPERATION, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-24T22:16:25,213 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37701 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=143 2024-11-24T22:16:25,213 DEBUG [PEWorker-3 {}] procedure.DeleteTableProcedure(115): Deleting regions from filesystem for pid=143, state=RUNNABLE:DELETE_TABLE_CLEAR_FS_LAYOUT, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-24T22:16:25,216 DEBUG [HFileArchiver-5 {}] backup.HFileArchiver(131): ARCHIVING hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/744bbec6934be304e01b009be760323d 2024-11-24T22:16:25,217 DEBUG [HFileArchiver-5 {}] backup.HFileArchiver(159): Archiving [FileablePath, hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/744bbec6934be304e01b009be760323d/A, FileablePath, hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/744bbec6934be304e01b009be760323d/B, FileablePath, hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/744bbec6934be304e01b009be760323d/C, FileablePath, hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/744bbec6934be304e01b009be760323d/recovered.edits] 2024-11-24T22:16:25,219 DEBUG [HFileArchiver-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/744bbec6934be304e01b009be760323d/A/0d73e3a2e443431baf4d1b1894f7323a to hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/archive/data/default/TestAcidGuarantees/744bbec6934be304e01b009be760323d/A/0d73e3a2e443431baf4d1b1894f7323a 2024-11-24T22:16:25,220 DEBUG [HFileArchiver-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/744bbec6934be304e01b009be760323d/A/21c5db2fe2e144398f211f2f114de18f to hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/archive/data/default/TestAcidGuarantees/744bbec6934be304e01b009be760323d/A/21c5db2fe2e144398f211f2f114de18f 2024-11-24T22:16:25,221 DEBUG [HFileArchiver-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/744bbec6934be304e01b009be760323d/A/2fd557a0f7c34347b9cbf6d42ee42a09 to hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/archive/data/default/TestAcidGuarantees/744bbec6934be304e01b009be760323d/A/2fd557a0f7c34347b9cbf6d42ee42a09 2024-11-24T22:16:25,223 DEBUG [HFileArchiver-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/744bbec6934be304e01b009be760323d/B/2aeb57cd3bf9441bb1932d9dcbba2ec2 to hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/archive/data/default/TestAcidGuarantees/744bbec6934be304e01b009be760323d/B/2aeb57cd3bf9441bb1932d9dcbba2ec2 2024-11-24T22:16:25,223 DEBUG [HFileArchiver-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/744bbec6934be304e01b009be760323d/B/a196a970e0dc4c8a965203b83b106cbf to hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/archive/data/default/TestAcidGuarantees/744bbec6934be304e01b009be760323d/B/a196a970e0dc4c8a965203b83b106cbf 2024-11-24T22:16:25,224 DEBUG [HFileArchiver-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/744bbec6934be304e01b009be760323d/B/ec23eec7790943728319381a913cd599 to hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/archive/data/default/TestAcidGuarantees/744bbec6934be304e01b009be760323d/B/ec23eec7790943728319381a913cd599 2024-11-24T22:16:25,226 DEBUG [HFileArchiver-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/744bbec6934be304e01b009be760323d/C/365e82b4a9704352abd426389dcfbb91 to hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/archive/data/default/TestAcidGuarantees/744bbec6934be304e01b009be760323d/C/365e82b4a9704352abd426389dcfbb91 2024-11-24T22:16:25,226 DEBUG [HFileArchiver-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/744bbec6934be304e01b009be760323d/C/80d409804e4a40d7a81a05a86f1b0f7f to hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/archive/data/default/TestAcidGuarantees/744bbec6934be304e01b009be760323d/C/80d409804e4a40d7a81a05a86f1b0f7f 2024-11-24T22:16:25,227 DEBUG [HFileArchiver-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/744bbec6934be304e01b009be760323d/C/ab4e14cb47dd453a8f163f214c641153 to hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/archive/data/default/TestAcidGuarantees/744bbec6934be304e01b009be760323d/C/ab4e14cb47dd453a8f163f214c641153 2024-11-24T22:16:25,229 DEBUG [HFileArchiver-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/744bbec6934be304e01b009be760323d/recovered.edits/461.seqid to hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/archive/data/default/TestAcidGuarantees/744bbec6934be304e01b009be760323d/recovered.edits/461.seqid 2024-11-24T22:16:25,229 DEBUG [HFileArchiver-5 {}] backup.HFileArchiver(610): Deleted hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/744bbec6934be304e01b009be760323d 2024-11-24T22:16:25,229 DEBUG [PEWorker-3 {}] procedure.DeleteTableProcedure(313): Archived TestAcidGuarantees regions 2024-11-24T22:16:25,231 DEBUG [PEWorker-3 {}] procedure.DeleteTableProcedure(120): Deleting regions from META for pid=143, state=RUNNABLE:DELETE_TABLE_REMOVE_FROM_META, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-24T22:16:25,233 WARN [PEWorker-3 {}] procedure.DeleteTableProcedure(370): Deleting some vestigial 1 rows of TestAcidGuarantees from hbase:meta 2024-11-24T22:16:25,234 DEBUG [PEWorker-3 {}] procedure.DeleteTableProcedure(407): Removing 'TestAcidGuarantees' descriptor. 2024-11-24T22:16:25,235 DEBUG [PEWorker-3 {}] procedure.DeleteTableProcedure(126): Deleting assignment state for pid=143, state=RUNNABLE:DELETE_TABLE_UNASSIGN_REGIONS, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-24T22:16:25,235 DEBUG [PEWorker-3 {}] procedure.DeleteTableProcedure(397): Removing 'TestAcidGuarantees' from region states. 2024-11-24T22:16:25,235 DEBUG [PEWorker-3 {}] assignment.RegionStateStore(723): Delete {"totalColumns":1,"row":"TestAcidGuarantees,,1732486556329.744bbec6934be304e01b009be760323d.","families":{"info":[{"qualifier":"","vlen":0,"tag":[],"timestamp":"1732486585235"}]},"ts":"9223372036854775807"} 2024-11-24T22:16:25,237 INFO [PEWorker-3 {}] assignment.RegionStateStore(562): Deleted 1 regions from META 2024-11-24T22:16:25,237 DEBUG [PEWorker-3 {}] assignment.RegionStateStore(563): Deleted regions: [{ENCODED => 744bbec6934be304e01b009be760323d, NAME => 'TestAcidGuarantees,,1732486556329.744bbec6934be304e01b009be760323d.', STARTKEY => '', ENDKEY => ''}] 2024-11-24T22:16:25,237 DEBUG [PEWorker-3 {}] procedure.DeleteTableProcedure(401): Marking 'TestAcidGuarantees' as deleted. 2024-11-24T22:16:25,237 DEBUG [PEWorker-3 {}] hbase.MetaTableAccessor(964): Delete {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":0,"tag":[],"timestamp":"1732486585237"}]},"ts":"9223372036854775807"} 2024-11-24T22:16:25,238 INFO [PEWorker-3 {}] hbase.MetaTableAccessor(867): Deleted table TestAcidGuarantees state from META 2024-11-24T22:16:25,239 DEBUG [PEWorker-3 {}] procedure.DeleteTableProcedure(133): Finished pid=143, state=RUNNABLE:DELETE_TABLE_POST_OPERATION, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-24T22:16:25,240 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=143, state=SUCCESS, hasLock=false; DeleteTableProcedure table=TestAcidGuarantees in 28 msec 2024-11-24T22:16:25,320 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37701 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=143 2024-11-24T22:16:25,321 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocator(219): Clear meta cache for TestAcidGuarantees 2024-11-24T22:16:25,321 INFO [RPCClient-NioEventLoopGroup-4-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: DELETE, Table Name: default:TestAcidGuarantees completed 2024-11-24T22:16:25,330 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: TestAcidGuaranteesWithEagerPolicy#testScanAtomicity Thread=248 (was 249), OpenFileDescriptor=449 (was 449), MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=330 (was 310) - SystemLoadAverage LEAK? -, ProcessCount=11 (was 11), AvailableMemoryMB=5785 (was 5852) 2024-11-24T22:16:25,338 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: TestAcidGuaranteesWithEagerPolicy#testMobGetAtomicity Thread=248, OpenFileDescriptor=449, MaxFileDescriptor=1048576, SystemLoadAverage=330, ProcessCount=11, AvailableMemoryMB=5785 2024-11-24T22:16:25,339 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37701 {}] util.TableDescriptorChecker(321): MEMSTORE_FLUSHSIZE for table descriptor or "hbase.hregion.memstore.flush.size" (131072) is too small, which might cause very frequent flushing. 2024-11-24T22:16:25,339 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37701 {}] master.HMaster$4(2454): Client=jenkins//172.17.0.2 create 'TestAcidGuarantees', {TABLE_ATTRIBUTES => {METADATA => {'hbase.hregion.compacting.memstore.type' => 'EAGER', 'hbase.hregion.memstore.mslab.enabled' => 'false', 'hbase.memstore.inmemoryflush.threshold.factor' => '0.9'}}}, {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-11-24T22:16:25,340 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37701 {}] procedure2.ProcedureExecutor(1139): Stored pid=144, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=false; CreateTableProcedure table=TestAcidGuarantees 2024-11-24T22:16:25,341 INFO [PEWorker-1 {}] procedure.CreateTableProcedure(89): pid=144, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_PRE_OPERATION 2024-11-24T22:16:25,341 DEBUG [PEWorker-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T22:16:25,341 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37701 {}] master.MasterRpcServices(787): Client=jenkins//172.17.0.2 procedure request for creating table: namespace: "default" qualifier: "TestAcidGuarantees" procId is: 144 2024-11-24T22:16:25,341 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37701 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=144 2024-11-24T22:16:25,341 INFO [PEWorker-1 {}] procedure.CreateTableProcedure(89): pid=144, state=RUNNABLE:CREATE_TABLE_WRITE_FS_LAYOUT, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_WRITE_FS_LAYOUT 2024-11-24T22:16:25,345 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45929 is added to blk_1073742383_1559 (size=1061) 2024-11-24T22:16:25,451 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37701 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=144 2024-11-24T22:16:25,661 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37701 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=144 2024-11-24T22:16:25,747 INFO [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(7572): creating {ENCODED => 8ca644d2cada6dc44b363b7383dcd4fe, NAME => 'TestAcidGuarantees,,1732486585339.8ca644d2cada6dc44b363b7383dcd4fe.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='TestAcidGuarantees', {TABLE_ATTRIBUTES => {METADATA => {'hbase.hregion.compacting.memstore.type' => 'EAGER', 'hbase.hregion.memstore.mslab.enabled' => 'false', 'hbase.memstore.inmemoryflush.threshold.factor' => '0.9', 'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc 2024-11-24T22:16:25,752 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45929 is added to blk_1073742384_1560 (size=53) 2024-11-24T22:16:25,971 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37701 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=144 2024-11-24T22:16:26,152 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(898): Instantiated TestAcidGuarantees,,1732486585339.8ca644d2cada6dc44b363b7383dcd4fe.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-24T22:16:26,153 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1722): Closing 8ca644d2cada6dc44b363b7383dcd4fe, disabling compactions & flushes 2024-11-24T22:16:26,153 INFO [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1755): Closing region TestAcidGuarantees,,1732486585339.8ca644d2cada6dc44b363b7383dcd4fe. 2024-11-24T22:16:26,153 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1776): Time limited wait for close lock on TestAcidGuarantees,,1732486585339.8ca644d2cada6dc44b363b7383dcd4fe. 2024-11-24T22:16:26,153 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1843): Acquired close lock on TestAcidGuarantees,,1732486585339.8ca644d2cada6dc44b363b7383dcd4fe. after waiting 0 ms 2024-11-24T22:16:26,153 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1853): Updates disabled for region TestAcidGuarantees,,1732486585339.8ca644d2cada6dc44b363b7383dcd4fe. 2024-11-24T22:16:26,153 INFO [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1973): Closed TestAcidGuarantees,,1732486585339.8ca644d2cada6dc44b363b7383dcd4fe. 2024-11-24T22:16:26,153 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1676): Region close journal for 8ca644d2cada6dc44b363b7383dcd4fe: Waiting for close lock at 1732486586153Disabling compacts and flushes for region at 1732486586153Disabling writes for close at 1732486586153Writing region close event to WAL at 1732486586153Closed at 1732486586153 2024-11-24T22:16:26,154 INFO [PEWorker-1 {}] procedure.CreateTableProcedure(89): pid=144, state=RUNNABLE:CREATE_TABLE_ADD_TO_META, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_ADD_TO_META 2024-11-24T22:16:26,154 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":2,"row":"TestAcidGuarantees,,1732486585339.8ca644d2cada6dc44b363b7383dcd4fe.","families":{"info":[{"qualifier":"regioninfo","vlen":52,"tag":[],"timestamp":"1732486586154"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1732486586154"}]},"ts":"1732486586154"} 2024-11-24T22:16:26,155 INFO [PEWorker-1 {}] hbase.MetaTableAccessor(832): Added 1 regions to meta. 2024-11-24T22:16:26,156 INFO [PEWorker-1 {}] procedure.CreateTableProcedure(89): pid=144, state=RUNNABLE:CREATE_TABLE_ASSIGN_REGIONS, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_ASSIGN_REGIONS 2024-11-24T22:16:26,156 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1732486586156"}]},"ts":"1732486586156"} 2024-11-24T22:16:26,157 INFO [PEWorker-1 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=ENABLING in hbase:meta 2024-11-24T22:16:26,157 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=145, ppid=144, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=8ca644d2cada6dc44b363b7383dcd4fe, ASSIGN}] 2024-11-24T22:16:26,157 INFO [PEWorker-2 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=145, ppid=144, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=8ca644d2cada6dc44b363b7383dcd4fe, ASSIGN 2024-11-24T22:16:26,158 INFO [PEWorker-2 {}] assignment.TransitRegionStateProcedure(269): Starting pid=145, ppid=144, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=8ca644d2cada6dc44b363b7383dcd4fe, ASSIGN; state=OFFLINE, location=bd6592c37bb2,40087,1732486423713; forceNewPlan=false, retain=false 2024-11-24T22:16:26,309 INFO [PEWorker-5 {}] assignment.RegionStateStore(223): pid=145 updating hbase:meta row=8ca644d2cada6dc44b363b7383dcd4fe, regionState=OPENING, regionLocation=bd6592c37bb2,40087,1732486423713 2024-11-24T22:16:26,310 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=145, ppid=144, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=8ca644d2cada6dc44b363b7383dcd4fe, ASSIGN because future has completed 2024-11-24T22:16:26,311 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=146, ppid=145, state=RUNNABLE, hasLock=false; OpenRegionProcedure 8ca644d2cada6dc44b363b7383dcd4fe, server=bd6592c37bb2,40087,1732486423713}] 2024-11-24T22:16:26,466 INFO [RS_OPEN_REGION-regionserver/bd6592c37bb2:0-0 {event_type=M_RS_OPEN_REGION, pid=146}] handler.AssignRegionHandler(132): Open TestAcidGuarantees,,1732486585339.8ca644d2cada6dc44b363b7383dcd4fe. 2024-11-24T22:16:26,466 DEBUG [RS_OPEN_REGION-regionserver/bd6592c37bb2:0-0 {event_type=M_RS_OPEN_REGION, pid=146}] regionserver.HRegion(7752): Opening region: {ENCODED => 8ca644d2cada6dc44b363b7383dcd4fe, NAME => 'TestAcidGuarantees,,1732486585339.8ca644d2cada6dc44b363b7383dcd4fe.', STARTKEY => '', ENDKEY => ''} 2024-11-24T22:16:26,466 DEBUG [RS_OPEN_REGION-regionserver/bd6592c37bb2:0-0 {event_type=M_RS_OPEN_REGION, pid=146}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table TestAcidGuarantees 8ca644d2cada6dc44b363b7383dcd4fe 2024-11-24T22:16:26,466 DEBUG [RS_OPEN_REGION-regionserver/bd6592c37bb2:0-0 {event_type=M_RS_OPEN_REGION, pid=146}] regionserver.HRegion(898): Instantiated TestAcidGuarantees,,1732486585339.8ca644d2cada6dc44b363b7383dcd4fe.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-24T22:16:26,467 DEBUG [RS_OPEN_REGION-regionserver/bd6592c37bb2:0-0 {event_type=M_RS_OPEN_REGION, pid=146}] regionserver.HRegion(7794): checking encryption for 8ca644d2cada6dc44b363b7383dcd4fe 2024-11-24T22:16:26,467 DEBUG [RS_OPEN_REGION-regionserver/bd6592c37bb2:0-0 {event_type=M_RS_OPEN_REGION, pid=146}] regionserver.HRegion(7797): checking classloading for 8ca644d2cada6dc44b363b7383dcd4fe 2024-11-24T22:16:26,468 INFO [StoreOpener-8ca644d2cada6dc44b363b7383dcd4fe-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family A of region 8ca644d2cada6dc44b363b7383dcd4fe 2024-11-24T22:16:26,469 INFO [StoreOpener-8ca644d2cada6dc44b363b7383dcd4fe-1 {}] regionserver.CompactingMemStore(122): Store=A, in-memory flush size threshold=38.40 KB, immutable segments index type=ARRAY_MAP, compactor=EAGER, pipelineThreshold=2, compactionCellMax=10 2024-11-24T22:16:26,469 INFO [StoreOpener-8ca644d2cada6dc44b363b7383dcd4fe-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 8ca644d2cada6dc44b363b7383dcd4fe columnFamilyName A 2024-11-24T22:16:26,469 DEBUG [StoreOpener-8ca644d2cada6dc44b363b7383dcd4fe-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T22:16:26,469 INFO [StoreOpener-8ca644d2cada6dc44b363b7383dcd4fe-1 {}] regionserver.HStore(327): Store=8ca644d2cada6dc44b363b7383dcd4fe/A, memstore type=CompactingMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-24T22:16:26,469 INFO [StoreOpener-8ca644d2cada6dc44b363b7383dcd4fe-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family B of region 8ca644d2cada6dc44b363b7383dcd4fe 2024-11-24T22:16:26,470 INFO [StoreOpener-8ca644d2cada6dc44b363b7383dcd4fe-1 {}] regionserver.CompactingMemStore(122): Store=B, in-memory flush size threshold=38.40 KB, immutable segments index type=ARRAY_MAP, compactor=EAGER, pipelineThreshold=2, compactionCellMax=10 2024-11-24T22:16:26,470 INFO [StoreOpener-8ca644d2cada6dc44b363b7383dcd4fe-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 8ca644d2cada6dc44b363b7383dcd4fe columnFamilyName B 2024-11-24T22:16:26,470 DEBUG [StoreOpener-8ca644d2cada6dc44b363b7383dcd4fe-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T22:16:26,471 INFO [StoreOpener-8ca644d2cada6dc44b363b7383dcd4fe-1 {}] regionserver.HStore(327): Store=8ca644d2cada6dc44b363b7383dcd4fe/B, memstore type=CompactingMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-24T22:16:26,471 INFO [StoreOpener-8ca644d2cada6dc44b363b7383dcd4fe-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family C of region 8ca644d2cada6dc44b363b7383dcd4fe 2024-11-24T22:16:26,471 INFO [StoreOpener-8ca644d2cada6dc44b363b7383dcd4fe-1 {}] regionserver.CompactingMemStore(122): Store=C, in-memory flush size threshold=38.40 KB, immutable segments index type=ARRAY_MAP, compactor=EAGER, pipelineThreshold=2, compactionCellMax=10 2024-11-24T22:16:26,471 INFO [StoreOpener-8ca644d2cada6dc44b363b7383dcd4fe-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 8ca644d2cada6dc44b363b7383dcd4fe columnFamilyName C 2024-11-24T22:16:26,471 DEBUG [StoreOpener-8ca644d2cada6dc44b363b7383dcd4fe-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T22:16:26,472 INFO [StoreOpener-8ca644d2cada6dc44b363b7383dcd4fe-1 {}] regionserver.HStore(327): Store=8ca644d2cada6dc44b363b7383dcd4fe/C, memstore type=CompactingMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-24T22:16:26,472 INFO [RS_OPEN_REGION-regionserver/bd6592c37bb2:0-0 {event_type=M_RS_OPEN_REGION, pid=146}] regionserver.HRegion(1212): Setting FlushNonSloppyStoresFirstPolicy for the region=TestAcidGuarantees,,1732486585339.8ca644d2cada6dc44b363b7383dcd4fe. 2024-11-24T22:16:26,472 DEBUG [RS_OPEN_REGION-regionserver/bd6592c37bb2:0-0 {event_type=M_RS_OPEN_REGION, pid=146}] regionserver.HRegion(1038): replaying wal for 8ca644d2cada6dc44b363b7383dcd4fe 2024-11-24T22:16:26,472 DEBUG [RS_OPEN_REGION-regionserver/bd6592c37bb2:0-0 {event_type=M_RS_OPEN_REGION, pid=146}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/8ca644d2cada6dc44b363b7383dcd4fe 2024-11-24T22:16:26,473 DEBUG [RS_OPEN_REGION-regionserver/bd6592c37bb2:0-0 {event_type=M_RS_OPEN_REGION, pid=146}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/8ca644d2cada6dc44b363b7383dcd4fe 2024-11-24T22:16:26,473 DEBUG [RS_OPEN_REGION-regionserver/bd6592c37bb2:0-0 {event_type=M_RS_OPEN_REGION, pid=146}] regionserver.HRegion(1048): stopping wal replay for 8ca644d2cada6dc44b363b7383dcd4fe 2024-11-24T22:16:26,473 DEBUG [RS_OPEN_REGION-regionserver/bd6592c37bb2:0-0 {event_type=M_RS_OPEN_REGION, pid=146}] regionserver.HRegion(1060): Cleaning up temporary data for 8ca644d2cada6dc44b363b7383dcd4fe 2024-11-24T22:16:26,474 DEBUG [RS_OPEN_REGION-regionserver/bd6592c37bb2:0-0 {event_type=M_RS_OPEN_REGION, pid=146}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table TestAcidGuarantees descriptor;using region.getMemStoreFlushHeapSize/# of families (16.0 M)) instead. 2024-11-24T22:16:26,475 DEBUG [RS_OPEN_REGION-regionserver/bd6592c37bb2:0-0 {event_type=M_RS_OPEN_REGION, pid=146}] regionserver.HRegion(1093): writing seq id for 8ca644d2cada6dc44b363b7383dcd4fe 2024-11-24T22:16:26,476 DEBUG [RS_OPEN_REGION-regionserver/bd6592c37bb2:0-0 {event_type=M_RS_OPEN_REGION, pid=146}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/8ca644d2cada6dc44b363b7383dcd4fe/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-24T22:16:26,476 INFO [RS_OPEN_REGION-regionserver/bd6592c37bb2:0-0 {event_type=M_RS_OPEN_REGION, pid=146}] regionserver.HRegion(1114): Opened 8ca644d2cada6dc44b363b7383dcd4fe; next sequenceid=2; ConstantSizeRegionSplitPolicy{desiredMaxFileSize=72183815, jitterRate=0.07562266290187836}, FlushLargeStoresPolicy{flushSizeLowerBound=16777216} 2024-11-24T22:16:26,476 DEBUG [RS_OPEN_REGION-regionserver/bd6592c37bb2:0-0 {event_type=M_RS_OPEN_REGION, pid=146}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 8ca644d2cada6dc44b363b7383dcd4fe 2024-11-24T22:16:26,477 DEBUG [RS_OPEN_REGION-regionserver/bd6592c37bb2:0-0 {event_type=M_RS_OPEN_REGION, pid=146}] regionserver.HRegion(1006): Region open journal for 8ca644d2cada6dc44b363b7383dcd4fe: Running coprocessor pre-open hook at 1732486586467Writing region info on filesystem at 1732486586467Initializing all the Stores at 1732486586467Instantiating store for column family {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732486586467Instantiating store for column family {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732486586467Instantiating store for column family {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732486586467Cleaning up temporary data from old regions at 1732486586473 (+6 ms)Running coprocessor post-open hooks at 1732486586476 (+3 ms)Region opened successfully at 1732486586477 (+1 ms) 2024-11-24T22:16:26,477 INFO [RS_OPEN_REGION-regionserver/bd6592c37bb2:0-0 {event_type=M_RS_OPEN_REGION, pid=146}] regionserver.HRegionServer(2236): Post open deploy tasks for TestAcidGuarantees,,1732486585339.8ca644d2cada6dc44b363b7383dcd4fe., pid=146, masterSystemTime=1732486586463 2024-11-24T22:16:26,478 DEBUG [RS_OPEN_REGION-regionserver/bd6592c37bb2:0-0 {event_type=M_RS_OPEN_REGION, pid=146}] regionserver.HRegionServer(2266): Finished post open deploy task for TestAcidGuarantees,,1732486585339.8ca644d2cada6dc44b363b7383dcd4fe. 2024-11-24T22:16:26,479 INFO [RS_OPEN_REGION-regionserver/bd6592c37bb2:0-0 {event_type=M_RS_OPEN_REGION, pid=146}] handler.AssignRegionHandler(153): Opened TestAcidGuarantees,,1732486585339.8ca644d2cada6dc44b363b7383dcd4fe. 2024-11-24T22:16:26,479 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=145 updating hbase:meta row=8ca644d2cada6dc44b363b7383dcd4fe, regionState=OPEN, openSeqNum=2, regionLocation=bd6592c37bb2,40087,1732486423713 2024-11-24T22:16:26,480 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=146, ppid=145, state=RUNNABLE, hasLock=false; OpenRegionProcedure 8ca644d2cada6dc44b363b7383dcd4fe, server=bd6592c37bb2,40087,1732486423713 because future has completed 2024-11-24T22:16:26,480 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37701 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=144 2024-11-24T22:16:26,482 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=146, resume processing ppid=145 2024-11-24T22:16:26,482 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=146, ppid=145, state=SUCCESS, hasLock=false; OpenRegionProcedure 8ca644d2cada6dc44b363b7383dcd4fe, server=bd6592c37bb2,40087,1732486423713 in 170 msec 2024-11-24T22:16:26,483 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=145, resume processing ppid=144 2024-11-24T22:16:26,483 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=145, ppid=144, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=8ca644d2cada6dc44b363b7383dcd4fe, ASSIGN in 325 msec 2024-11-24T22:16:26,484 INFO [PEWorker-4 {}] procedure.CreateTableProcedure(89): pid=144, state=RUNNABLE:CREATE_TABLE_UPDATE_DESC_CACHE, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_UPDATE_DESC_CACHE 2024-11-24T22:16:26,484 DEBUG [PEWorker-4 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1732486586484"}]},"ts":"1732486586484"} 2024-11-24T22:16:26,485 INFO [PEWorker-4 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=ENABLED in hbase:meta 2024-11-24T22:16:26,485 INFO [PEWorker-4 {}] procedure.CreateTableProcedure(89): pid=144, state=RUNNABLE:CREATE_TABLE_POST_OPERATION, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_POST_OPERATION 2024-11-24T22:16:26,486 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=144, state=SUCCESS, hasLock=false; CreateTableProcedure table=TestAcidGuarantees in 1.1460 sec 2024-11-24T22:16:27,491 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37701 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=144 2024-11-24T22:16:27,491 INFO [RPCClient-NioEventLoopGroup-4-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: CREATE, Table Name: default:TestAcidGuarantees completed 2024-11-24T22:16:27,492 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@a04de68, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-24T22:16:27,492 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request bd6592c37bb2,37701,-1 for getting cluster id 2024-11-24T22:16:27,493 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-24T22:16:27,493 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '3c461ac1-06fb-4739-8702-6368e685b03c' 2024-11-24T22:16:27,494 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-24T22:16:27,494 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "3c461ac1-06fb-4739-8702-6368e685b03c" 2024-11-24T22:16:27,494 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@6f4afbd1, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-24T22:16:27,494 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [bd6592c37bb2,37701,-1] 2024-11-24T22:16:27,494 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-24T22:16:27,494 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-24T22:16:27,495 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:46748, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-24T22:16:27,495 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@3c0365f3, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-24T22:16:27,496 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-24T22:16:27,496 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=bd6592c37bb2,40087,1732486423713, seqNum=-1] 2024-11-24T22:16:27,496 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-24T22:16:27,497 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:48962, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-24T22:16:27,498 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching master stub from registry 2024-11-24T22:16:27,498 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncConnectionImpl(321): The fetched master address is bd6592c37bb2,37701,1732486422992 2024-11-24T22:16:27,498 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.ConnectionUtils(555): The fetched master stub is org.apache.hadoop.hbase.shaded.protobuf.generated.MasterProtos$MasterService$Stub@7ede8f3e 2024-11-24T22:16:27,498 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=MasterService, sasl=false 2024-11-24T22:16:27,499 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:46764, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=MasterService 2024-11-24T22:16:27,501 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37701 {}] util.TableDescriptorChecker(321): MEMSTORE_FLUSHSIZE for table descriptor or "hbase.hregion.memstore.flush.size" (131072) is too small, which might cause very frequent flushing. 2024-11-24T22:16:27,501 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37701 {}] master.HMaster$14(2863): Client=jenkins//172.17.0.2 modify table TestAcidGuarantees from 'TestAcidGuarantees', {TABLE_ATTRIBUTES => {METADATA => {'hbase.hregion.compacting.memstore.type' => 'EAGER', 'hbase.hregion.memstore.mslab.enabled' => 'false', 'hbase.memstore.inmemoryflush.threshold.factor' => '0.9', 'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} to 'TestAcidGuarantees', {TABLE_ATTRIBUTES => {METADATA => {'hbase.hregion.compacting.memstore.type' => 'EAGER', 'hbase.hregion.memstore.mslab.enabled' => 'false', 'hbase.memstore.inmemoryflush.threshold.factor' => '0.9', 'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', MOB_THRESHOLD => '4', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', IS_MOB => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-11-24T22:16:27,502 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37701 {}] procedure2.ProcedureExecutor(1139): Stored pid=147, state=RUNNABLE:MODIFY_TABLE_PREPARE, hasLock=false; ModifyTableProcedure table=TestAcidGuarantees 2024-11-24T22:16:27,510 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45929 is added to blk_1073742385_1561 (size=1097) 2024-11-24T22:16:27,911 DEBUG [PEWorker-3 {}] util.FSTableDescriptors(591): Deleted hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/.tabledesc/.tableinfo.0000000001.1061 2024-11-24T22:16:27,912 INFO [PEWorker-3 {}] util.FSTableDescriptors(361): Updated tableinfo=hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/.tabledesc/.tableinfo.0000000002.1097 2024-11-24T22:16:27,914 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=148, ppid=147, state=RUNNABLE:REOPEN_TABLE_REGIONS_GET_REGIONS, hasLock=false; ReopenTableRegionsProcedure table=TestAcidGuarantees}] 2024-11-24T22:16:27,915 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=149, ppid=148, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=8ca644d2cada6dc44b363b7383dcd4fe, REOPEN/MOVE}] 2024-11-24T22:16:27,916 INFO [PEWorker-2 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=149, ppid=148, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=8ca644d2cada6dc44b363b7383dcd4fe, REOPEN/MOVE 2024-11-24T22:16:27,916 INFO [PEWorker-2 {}] assignment.RegionStateStore(223): pid=149 updating hbase:meta row=8ca644d2cada6dc44b363b7383dcd4fe, regionState=CLOSING, regionLocation=bd6592c37bb2,40087,1732486423713 2024-11-24T22:16:27,918 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=149, ppid=148, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=8ca644d2cada6dc44b363b7383dcd4fe, REOPEN/MOVE because future has completed 2024-11-24T22:16:27,918 DEBUG [PEWorker-5 {}] assignment.TransitRegionStateProcedure(375): Close region: isSplit: false: evictOnSplit: true: evictOnClose: false 2024-11-24T22:16:27,918 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=150, ppid=149, state=RUNNABLE, hasLock=false; CloseRegionProcedure 8ca644d2cada6dc44b363b7383dcd4fe, server=bd6592c37bb2,40087,1732486423713}] 2024-11-24T22:16:28,070 INFO [RS_CLOSE_REGION-regionserver/bd6592c37bb2:0-0 {event_type=M_RS_CLOSE_REGION, pid=150}] handler.UnassignRegionHandler(122): Close 8ca644d2cada6dc44b363b7383dcd4fe 2024-11-24T22:16:28,070 DEBUG [RS_CLOSE_REGION-regionserver/bd6592c37bb2:0-0 {event_type=M_RS_CLOSE_REGION, pid=150}] handler.UnassignRegionHandler(136): Unassign region: split region: false: evictCache: false 2024-11-24T22:16:28,070 DEBUG [RS_CLOSE_REGION-regionserver/bd6592c37bb2:0-0 {event_type=M_RS_CLOSE_REGION, pid=150}] regionserver.HRegion(1722): Closing 8ca644d2cada6dc44b363b7383dcd4fe, disabling compactions & flushes 2024-11-24T22:16:28,070 INFO [RS_CLOSE_REGION-regionserver/bd6592c37bb2:0-0 {event_type=M_RS_CLOSE_REGION, pid=150}] regionserver.HRegion(1755): Closing region TestAcidGuarantees,,1732486585339.8ca644d2cada6dc44b363b7383dcd4fe. 2024-11-24T22:16:28,070 DEBUG [RS_CLOSE_REGION-regionserver/bd6592c37bb2:0-0 {event_type=M_RS_CLOSE_REGION, pid=150}] regionserver.HRegion(1776): Time limited wait for close lock on TestAcidGuarantees,,1732486585339.8ca644d2cada6dc44b363b7383dcd4fe. 2024-11-24T22:16:28,070 DEBUG [RS_CLOSE_REGION-regionserver/bd6592c37bb2:0-0 {event_type=M_RS_CLOSE_REGION, pid=150}] regionserver.HRegion(1843): Acquired close lock on TestAcidGuarantees,,1732486585339.8ca644d2cada6dc44b363b7383dcd4fe. after waiting 0 ms 2024-11-24T22:16:28,070 DEBUG [RS_CLOSE_REGION-regionserver/bd6592c37bb2:0-0 {event_type=M_RS_CLOSE_REGION, pid=150}] regionserver.HRegion(1853): Updates disabled for region TestAcidGuarantees,,1732486585339.8ca644d2cada6dc44b363b7383dcd4fe. 2024-11-24T22:16:28,074 DEBUG [RS_CLOSE_REGION-regionserver/bd6592c37bb2:0-0 {event_type=M_RS_CLOSE_REGION, pid=150}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/8ca644d2cada6dc44b363b7383dcd4fe/recovered.edits/4.seqid, newMaxSeqId=4, maxSeqId=1 2024-11-24T22:16:28,074 INFO [RS_CLOSE_REGION-regionserver/bd6592c37bb2:0-0 {event_type=M_RS_CLOSE_REGION, pid=150}] regionserver.HRegion(1973): Closed TestAcidGuarantees,,1732486585339.8ca644d2cada6dc44b363b7383dcd4fe. 2024-11-24T22:16:28,074 DEBUG [RS_CLOSE_REGION-regionserver/bd6592c37bb2:0-0 {event_type=M_RS_CLOSE_REGION, pid=150}] regionserver.HRegion(1676): Region close journal for 8ca644d2cada6dc44b363b7383dcd4fe: Waiting for close lock at 1732486588070Running coprocessor pre-close hooks at 1732486588070Disabling compacts and flushes for region at 1732486588070Disabling writes for close at 1732486588070Writing region close event to WAL at 1732486588071 (+1 ms)Running coprocessor post-close hooks at 1732486588074 (+3 ms)Closed at 1732486588074 2024-11-24T22:16:28,074 WARN [RS_CLOSE_REGION-regionserver/bd6592c37bb2:0-0 {event_type=M_RS_CLOSE_REGION, pid=150}] regionserver.HRegionServer(3299): Not adding moved region record: 8ca644d2cada6dc44b363b7383dcd4fe to self. 2024-11-24T22:16:28,076 INFO [RS_CLOSE_REGION-regionserver/bd6592c37bb2:0-0 {event_type=M_RS_CLOSE_REGION, pid=150}] handler.UnassignRegionHandler(157): Closed 8ca644d2cada6dc44b363b7383dcd4fe 2024-11-24T22:16:28,076 INFO [PEWorker-3 {}] assignment.RegionStateStore(223): pid=149 updating hbase:meta row=8ca644d2cada6dc44b363b7383dcd4fe, regionState=CLOSED 2024-11-24T22:16:28,077 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=150, ppid=149, state=RUNNABLE, hasLock=false; CloseRegionProcedure 8ca644d2cada6dc44b363b7383dcd4fe, server=bd6592c37bb2,40087,1732486423713 because future has completed 2024-11-24T22:16:28,079 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=150, resume processing ppid=149 2024-11-24T22:16:28,079 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=150, ppid=149, state=SUCCESS, hasLock=false; CloseRegionProcedure 8ca644d2cada6dc44b363b7383dcd4fe, server=bd6592c37bb2,40087,1732486423713 in 160 msec 2024-11-24T22:16:28,079 INFO [PEWorker-2 {}] assignment.TransitRegionStateProcedure(269): Starting pid=149, ppid=148, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=8ca644d2cada6dc44b363b7383dcd4fe, REOPEN/MOVE; state=CLOSED, location=bd6592c37bb2,40087,1732486423713; forceNewPlan=false, retain=true 2024-11-24T22:16:28,230 INFO [PEWorker-4 {}] assignment.RegionStateStore(223): pid=149 updating hbase:meta row=8ca644d2cada6dc44b363b7383dcd4fe, regionState=OPENING, regionLocation=bd6592c37bb2,40087,1732486423713 2024-11-24T22:16:28,231 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=149, ppid=148, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=8ca644d2cada6dc44b363b7383dcd4fe, REOPEN/MOVE because future has completed 2024-11-24T22:16:28,231 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=151, ppid=149, state=RUNNABLE, hasLock=false; OpenRegionProcedure 8ca644d2cada6dc44b363b7383dcd4fe, server=bd6592c37bb2,40087,1732486423713}] 2024-11-24T22:16:28,385 INFO [RS_OPEN_REGION-regionserver/bd6592c37bb2:0-0 {event_type=M_RS_OPEN_REGION, pid=151}] handler.AssignRegionHandler(132): Open TestAcidGuarantees,,1732486585339.8ca644d2cada6dc44b363b7383dcd4fe. 2024-11-24T22:16:28,385 DEBUG [RS_OPEN_REGION-regionserver/bd6592c37bb2:0-0 {event_type=M_RS_OPEN_REGION, pid=151}] regionserver.HRegion(7752): Opening region: {ENCODED => 8ca644d2cada6dc44b363b7383dcd4fe, NAME => 'TestAcidGuarantees,,1732486585339.8ca644d2cada6dc44b363b7383dcd4fe.', STARTKEY => '', ENDKEY => ''} 2024-11-24T22:16:28,386 DEBUG [RS_OPEN_REGION-regionserver/bd6592c37bb2:0-0 {event_type=M_RS_OPEN_REGION, pid=151}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table TestAcidGuarantees 8ca644d2cada6dc44b363b7383dcd4fe 2024-11-24T22:16:28,386 DEBUG [RS_OPEN_REGION-regionserver/bd6592c37bb2:0-0 {event_type=M_RS_OPEN_REGION, pid=151}] regionserver.HRegion(898): Instantiated TestAcidGuarantees,,1732486585339.8ca644d2cada6dc44b363b7383dcd4fe.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-24T22:16:28,386 DEBUG [RS_OPEN_REGION-regionserver/bd6592c37bb2:0-0 {event_type=M_RS_OPEN_REGION, pid=151}] regionserver.HRegion(7794): checking encryption for 8ca644d2cada6dc44b363b7383dcd4fe 2024-11-24T22:16:28,386 DEBUG [RS_OPEN_REGION-regionserver/bd6592c37bb2:0-0 {event_type=M_RS_OPEN_REGION, pid=151}] regionserver.HRegion(7797): checking classloading for 8ca644d2cada6dc44b363b7383dcd4fe 2024-11-24T22:16:28,387 INFO [StoreOpener-8ca644d2cada6dc44b363b7383dcd4fe-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family A of region 8ca644d2cada6dc44b363b7383dcd4fe 2024-11-24T22:16:28,387 INFO [StoreOpener-8ca644d2cada6dc44b363b7383dcd4fe-1 {}] regionserver.CompactingMemStore(122): Store=A, in-memory flush size threshold=38.40 KB, immutable segments index type=ARRAY_MAP, compactor=EAGER, pipelineThreshold=2, compactionCellMax=10 2024-11-24T22:16:28,387 INFO [StoreOpener-8ca644d2cada6dc44b363b7383dcd4fe-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 8ca644d2cada6dc44b363b7383dcd4fe columnFamilyName A 2024-11-24T22:16:28,388 DEBUG [StoreOpener-8ca644d2cada6dc44b363b7383dcd4fe-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T22:16:28,389 INFO [StoreOpener-8ca644d2cada6dc44b363b7383dcd4fe-1 {}] regionserver.HStore(327): Store=8ca644d2cada6dc44b363b7383dcd4fe/A, memstore type=CompactingMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-24T22:16:28,389 INFO [StoreOpener-8ca644d2cada6dc44b363b7383dcd4fe-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family B of region 8ca644d2cada6dc44b363b7383dcd4fe 2024-11-24T22:16:28,389 INFO [StoreOpener-8ca644d2cada6dc44b363b7383dcd4fe-1 {}] regionserver.CompactingMemStore(122): Store=B, in-memory flush size threshold=38.40 KB, immutable segments index type=ARRAY_MAP, compactor=EAGER, pipelineThreshold=2, compactionCellMax=10 2024-11-24T22:16:28,389 INFO [StoreOpener-8ca644d2cada6dc44b363b7383dcd4fe-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 8ca644d2cada6dc44b363b7383dcd4fe columnFamilyName B 2024-11-24T22:16:28,389 DEBUG [StoreOpener-8ca644d2cada6dc44b363b7383dcd4fe-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T22:16:28,390 INFO [StoreOpener-8ca644d2cada6dc44b363b7383dcd4fe-1 {}] regionserver.HStore(327): Store=8ca644d2cada6dc44b363b7383dcd4fe/B, memstore type=CompactingMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-24T22:16:28,390 INFO [StoreOpener-8ca644d2cada6dc44b363b7383dcd4fe-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family C of region 8ca644d2cada6dc44b363b7383dcd4fe 2024-11-24T22:16:28,390 INFO [StoreOpener-8ca644d2cada6dc44b363b7383dcd4fe-1 {}] regionserver.CompactingMemStore(122): Store=C, in-memory flush size threshold=38.40 KB, immutable segments index type=ARRAY_MAP, compactor=EAGER, pipelineThreshold=2, compactionCellMax=10 2024-11-24T22:16:28,390 INFO [StoreOpener-8ca644d2cada6dc44b363b7383dcd4fe-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 8ca644d2cada6dc44b363b7383dcd4fe columnFamilyName C 2024-11-24T22:16:28,390 DEBUG [StoreOpener-8ca644d2cada6dc44b363b7383dcd4fe-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T22:16:28,390 INFO [StoreOpener-8ca644d2cada6dc44b363b7383dcd4fe-1 {}] regionserver.HStore(327): Store=8ca644d2cada6dc44b363b7383dcd4fe/C, memstore type=CompactingMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-24T22:16:28,391 INFO [RS_OPEN_REGION-regionserver/bd6592c37bb2:0-0 {event_type=M_RS_OPEN_REGION, pid=151}] regionserver.HRegion(1212): Setting FlushNonSloppyStoresFirstPolicy for the region=TestAcidGuarantees,,1732486585339.8ca644d2cada6dc44b363b7383dcd4fe. 2024-11-24T22:16:28,391 DEBUG [RS_OPEN_REGION-regionserver/bd6592c37bb2:0-0 {event_type=M_RS_OPEN_REGION, pid=151}] regionserver.HRegion(1038): replaying wal for 8ca644d2cada6dc44b363b7383dcd4fe 2024-11-24T22:16:28,391 DEBUG [RS_OPEN_REGION-regionserver/bd6592c37bb2:0-0 {event_type=M_RS_OPEN_REGION, pid=151}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/8ca644d2cada6dc44b363b7383dcd4fe 2024-11-24T22:16:28,392 DEBUG [RS_OPEN_REGION-regionserver/bd6592c37bb2:0-0 {event_type=M_RS_OPEN_REGION, pid=151}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/8ca644d2cada6dc44b363b7383dcd4fe 2024-11-24T22:16:28,392 DEBUG [RS_OPEN_REGION-regionserver/bd6592c37bb2:0-0 {event_type=M_RS_OPEN_REGION, pid=151}] regionserver.HRegion(1048): stopping wal replay for 8ca644d2cada6dc44b363b7383dcd4fe 2024-11-24T22:16:28,392 DEBUG [RS_OPEN_REGION-regionserver/bd6592c37bb2:0-0 {event_type=M_RS_OPEN_REGION, pid=151}] regionserver.HRegion(1060): Cleaning up temporary data for 8ca644d2cada6dc44b363b7383dcd4fe 2024-11-24T22:16:28,393 DEBUG [RS_OPEN_REGION-regionserver/bd6592c37bb2:0-0 {event_type=M_RS_OPEN_REGION, pid=151}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table TestAcidGuarantees descriptor;using region.getMemStoreFlushHeapSize/# of families (16.0 M)) instead. 2024-11-24T22:16:28,394 DEBUG [RS_OPEN_REGION-regionserver/bd6592c37bb2:0-0 {event_type=M_RS_OPEN_REGION, pid=151}] regionserver.HRegion(1093): writing seq id for 8ca644d2cada6dc44b363b7383dcd4fe 2024-11-24T22:16:28,394 INFO [RS_OPEN_REGION-regionserver/bd6592c37bb2:0-0 {event_type=M_RS_OPEN_REGION, pid=151}] regionserver.HRegion(1114): Opened 8ca644d2cada6dc44b363b7383dcd4fe; next sequenceid=5; ConstantSizeRegionSplitPolicy{desiredMaxFileSize=61598741, jitterRate=-0.08210723102092743}, FlushLargeStoresPolicy{flushSizeLowerBound=16777216} 2024-11-24T22:16:28,394 DEBUG [RS_OPEN_REGION-regionserver/bd6592c37bb2:0-0 {event_type=M_RS_OPEN_REGION, pid=151}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 8ca644d2cada6dc44b363b7383dcd4fe 2024-11-24T22:16:28,395 DEBUG [RS_OPEN_REGION-regionserver/bd6592c37bb2:0-0 {event_type=M_RS_OPEN_REGION, pid=151}] regionserver.HRegion(1006): Region open journal for 8ca644d2cada6dc44b363b7383dcd4fe: Running coprocessor pre-open hook at 1732486588386Writing region info on filesystem at 1732486588386Initializing all the Stores at 1732486588386Instantiating store for column family {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', MOB_THRESHOLD => '4', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', IS_MOB => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732486588386Instantiating store for column family {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732486588387 (+1 ms)Instantiating store for column family {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732486588387Cleaning up temporary data from old regions at 1732486588392 (+5 ms)Running coprocessor post-open hooks at 1732486588394 (+2 ms)Region opened successfully at 1732486588395 (+1 ms) 2024-11-24T22:16:28,395 INFO [RS_OPEN_REGION-regionserver/bd6592c37bb2:0-0 {event_type=M_RS_OPEN_REGION, pid=151}] regionserver.HRegionServer(2236): Post open deploy tasks for TestAcidGuarantees,,1732486585339.8ca644d2cada6dc44b363b7383dcd4fe., pid=151, masterSystemTime=1732486588383 2024-11-24T22:16:28,396 DEBUG [RS_OPEN_REGION-regionserver/bd6592c37bb2:0-0 {event_type=M_RS_OPEN_REGION, pid=151}] regionserver.HRegionServer(2266): Finished post open deploy task for TestAcidGuarantees,,1732486585339.8ca644d2cada6dc44b363b7383dcd4fe. 2024-11-24T22:16:28,396 INFO [RS_OPEN_REGION-regionserver/bd6592c37bb2:0-0 {event_type=M_RS_OPEN_REGION, pid=151}] handler.AssignRegionHandler(153): Opened TestAcidGuarantees,,1732486585339.8ca644d2cada6dc44b363b7383dcd4fe. 2024-11-24T22:16:28,397 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=149 updating hbase:meta row=8ca644d2cada6dc44b363b7383dcd4fe, regionState=OPEN, openSeqNum=5, regionLocation=bd6592c37bb2,40087,1732486423713 2024-11-24T22:16:28,398 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=151, ppid=149, state=RUNNABLE, hasLock=false; OpenRegionProcedure 8ca644d2cada6dc44b363b7383dcd4fe, server=bd6592c37bb2,40087,1732486423713 because future has completed 2024-11-24T22:16:28,400 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=151, resume processing ppid=149 2024-11-24T22:16:28,400 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=151, ppid=149, state=SUCCESS, hasLock=false; OpenRegionProcedure 8ca644d2cada6dc44b363b7383dcd4fe, server=bd6592c37bb2,40087,1732486423713 in 168 msec 2024-11-24T22:16:28,401 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=149, resume processing ppid=148 2024-11-24T22:16:28,401 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=149, ppid=148, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=8ca644d2cada6dc44b363b7383dcd4fe, REOPEN/MOVE in 485 msec 2024-11-24T22:16:28,404 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=148, resume processing ppid=147 2024-11-24T22:16:28,404 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=148, ppid=147, state=SUCCESS, hasLock=false; ReopenTableRegionsProcedure table=TestAcidGuarantees in 488 msec 2024-11-24T22:16:28,405 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=147, state=SUCCESS, hasLock=false; ModifyTableProcedure table=TestAcidGuarantees in 903 msec 2024-11-24T22:16:28,406 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37701 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=147 2024-11-24T22:16:28,406 INFO [RPCClient-NioEventLoopGroup-4-12 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: MODIFY_COLUMN_FAMILY, Table Name: default:TestAcidGuarantees completed 2024-11-24T22:16:28,407 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@53954261, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-24T22:16:28,407 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request bd6592c37bb2,37701,-1 for getting cluster id 2024-11-24T22:16:28,407 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-24T22:16:28,408 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '3c461ac1-06fb-4739-8702-6368e685b03c' 2024-11-24T22:16:28,408 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-24T22:16:28,408 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "3c461ac1-06fb-4739-8702-6368e685b03c" 2024-11-24T22:16:28,408 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@3ff8e76b, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-24T22:16:28,408 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [bd6592c37bb2,37701,-1] 2024-11-24T22:16:28,408 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-24T22:16:28,408 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-24T22:16:28,409 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:46782, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-24T22:16:28,410 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@5a971423, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-24T22:16:28,410 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@3e2aa61c, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-24T22:16:28,410 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request bd6592c37bb2,37701,-1 for getting cluster id 2024-11-24T22:16:28,410 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-24T22:16:28,411 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '3c461ac1-06fb-4739-8702-6368e685b03c' 2024-11-24T22:16:28,411 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-24T22:16:28,411 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "3c461ac1-06fb-4739-8702-6368e685b03c" 2024-11-24T22:16:28,411 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@4e243f4f, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-24T22:16:28,411 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [bd6592c37bb2,37701,-1] 2024-11-24T22:16:28,411 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-24T22:16:28,411 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-24T22:16:28,412 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:46806, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-24T22:16:28,412 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@63e725c0, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-24T22:16:28,413 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@5ca6c80d, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-24T22:16:28,413 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request bd6592c37bb2,37701,-1 for getting cluster id 2024-11-24T22:16:28,413 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-24T22:16:28,413 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '3c461ac1-06fb-4739-8702-6368e685b03c' 2024-11-24T22:16:28,414 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-24T22:16:28,414 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "3c461ac1-06fb-4739-8702-6368e685b03c" 2024-11-24T22:16:28,414 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@a329293, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-24T22:16:28,414 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [bd6592c37bb2,37701,-1] 2024-11-24T22:16:28,414 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-24T22:16:28,414 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-24T22:16:28,415 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:46820, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-24T22:16:28,415 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@a49ca28, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-24T22:16:28,416 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@14afc90b, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-24T22:16:28,416 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request bd6592c37bb2,37701,-1 for getting cluster id 2024-11-24T22:16:28,416 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-24T22:16:28,416 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '3c461ac1-06fb-4739-8702-6368e685b03c' 2024-11-24T22:16:28,416 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-24T22:16:28,416 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "3c461ac1-06fb-4739-8702-6368e685b03c" 2024-11-24T22:16:28,417 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@63af6e54, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-24T22:16:28,417 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [bd6592c37bb2,37701,-1] 2024-11-24T22:16:28,417 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-24T22:16:28,417 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-24T22:16:28,417 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:46846, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-24T22:16:28,418 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@63624631, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-24T22:16:28,418 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@1cb57557, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-24T22:16:28,418 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request bd6592c37bb2,37701,-1 for getting cluster id 2024-11-24T22:16:28,418 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-24T22:16:28,419 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '3c461ac1-06fb-4739-8702-6368e685b03c' 2024-11-24T22:16:28,419 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-24T22:16:28,419 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "3c461ac1-06fb-4739-8702-6368e685b03c" 2024-11-24T22:16:28,419 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@1bbe16d5, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-24T22:16:28,419 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [bd6592c37bb2,37701,-1] 2024-11-24T22:16:28,419 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-24T22:16:28,419 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-24T22:16:28,420 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:46860, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-24T22:16:28,420 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@1036f5ea, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-24T22:16:28,421 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@99b73f, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-24T22:16:28,421 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request bd6592c37bb2,37701,-1 for getting cluster id 2024-11-24T22:16:28,421 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-24T22:16:28,421 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '3c461ac1-06fb-4739-8702-6368e685b03c' 2024-11-24T22:16:28,421 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-24T22:16:28,421 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "3c461ac1-06fb-4739-8702-6368e685b03c" 2024-11-24T22:16:28,422 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@21252a89, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-24T22:16:28,422 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [bd6592c37bb2,37701,-1] 2024-11-24T22:16:28,422 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-24T22:16:28,422 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-24T22:16:28,422 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:46868, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-24T22:16:28,423 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@2bfc5f21, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-24T22:16:28,423 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@af69a6d, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-24T22:16:28,423 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request bd6592c37bb2,37701,-1 for getting cluster id 2024-11-24T22:16:28,423 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-24T22:16:28,424 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '3c461ac1-06fb-4739-8702-6368e685b03c' 2024-11-24T22:16:28,424 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-24T22:16:28,424 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "3c461ac1-06fb-4739-8702-6368e685b03c" 2024-11-24T22:16:28,424 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@688e55a3, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-24T22:16:28,424 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [bd6592c37bb2,37701,-1] 2024-11-24T22:16:28,424 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-24T22:16:28,425 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-24T22:16:28,425 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:46890, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-24T22:16:28,426 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@2e8b71c0, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-24T22:16:28,426 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@4fd90259, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-24T22:16:28,426 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request bd6592c37bb2,37701,-1 for getting cluster id 2024-11-24T22:16:28,426 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-24T22:16:28,427 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '3c461ac1-06fb-4739-8702-6368e685b03c' 2024-11-24T22:16:28,427 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-24T22:16:28,427 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "3c461ac1-06fb-4739-8702-6368e685b03c" 2024-11-24T22:16:28,427 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@58429940, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-24T22:16:28,427 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [bd6592c37bb2,37701,-1] 2024-11-24T22:16:28,428 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-24T22:16:28,428 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-24T22:16:28,428 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:46910, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-24T22:16:28,429 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@70fd2f9c, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-24T22:16:28,429 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@594d5ccc, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-24T22:16:28,429 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request bd6592c37bb2,37701,-1 for getting cluster id 2024-11-24T22:16:28,429 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-24T22:16:28,430 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '3c461ac1-06fb-4739-8702-6368e685b03c' 2024-11-24T22:16:28,430 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-24T22:16:28,430 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "3c461ac1-06fb-4739-8702-6368e685b03c" 2024-11-24T22:16:28,430 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@92a0bcd, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-24T22:16:28,430 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [bd6592c37bb2,37701,-1] 2024-11-24T22:16:28,430 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-24T22:16:28,430 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-24T22:16:28,431 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:46932, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-24T22:16:28,432 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@e8405b0, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-24T22:16:28,432 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@267cfa21, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-24T22:16:28,432 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request bd6592c37bb2,37701,-1 for getting cluster id 2024-11-24T22:16:28,432 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-24T22:16:28,433 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '3c461ac1-06fb-4739-8702-6368e685b03c' 2024-11-24T22:16:28,433 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-24T22:16:28,433 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "3c461ac1-06fb-4739-8702-6368e685b03c" 2024-11-24T22:16:28,433 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@76896e0f, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-24T22:16:28,433 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [bd6592c37bb2,37701,-1] 2024-11-24T22:16:28,433 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-24T22:16:28,433 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-24T22:16:28,434 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:46958, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-24T22:16:28,434 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@3b4e4411, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-24T22:16:28,435 DEBUG [Thread-2509 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-24T22:16:28,435 DEBUG [Thread-2513 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-24T22:16:28,435 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37701 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-24T22:16:28,435 DEBUG [Thread-2507 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-24T22:16:28,435 DEBUG [Thread-2498 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-24T22:16:28,436 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=bd6592c37bb2,40087,1732486423713, seqNum=-1] 2024-11-24T22:16:28,436 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=bd6592c37bb2,40087,1732486423713, seqNum=-1] 2024-11-24T22:16:28,436 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-24T22:16:28,436 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-24T22:16:28,436 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37701 {}] procedure2.ProcedureExecutor(1139): Stored pid=152, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=152, table=TestAcidGuarantees 2024-11-24T22:16:28,436 DEBUG [Thread-2502 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-24T22:16:28,436 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=bd6592c37bb2,40087,1732486423713, seqNum=-1] 2024-11-24T22:16:28,436 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=bd6592c37bb2,40087,1732486423713, seqNum=-1] 2024-11-24T22:16:28,436 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-24T22:16:28,437 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-24T22:16:28,437 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:48968, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-24T22:16:28,437 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=bd6592c37bb2,40087,1732486423713, seqNum=-1] 2024-11-24T22:16:28,437 DEBUG [Thread-2505 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-24T22:16:28,437 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-24T22:16:28,437 DEBUG [Thread-2494 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-24T22:16:28,437 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:48972, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-24T22:16:28,437 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=152, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=152, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-24T22:16:28,437 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:48974, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-24T22:16:28,438 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:49002, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-24T22:16:28,438 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=bd6592c37bb2,40087,1732486423713, seqNum=-1] 2024-11-24T22:16:28,438 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=bd6592c37bb2,40087,1732486423713, seqNum=-1] 2024-11-24T22:16:28,438 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-24T22:16:28,438 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=152, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=152, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-24T22:16:28,438 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-24T22:16:28,438 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=153, ppid=152, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-24T22:16:28,439 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:49018, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-24T22:16:28,439 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_2', locateType=CURRENT is [region=TestAcidGuarantees,,1732486585339.8ca644d2cada6dc44b363b7383dcd4fe., hostname=bd6592c37bb2,40087,1732486423713, seqNum=5] 2024-11-24T22:16:28,439 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_1', locateType=CURRENT is [region=TestAcidGuarantees,,1732486585339.8ca644d2cada6dc44b363b7383dcd4fe., hostname=bd6592c37bb2,40087,1732486423713, seqNum=5] 2024-11-24T22:16:28,439 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:48986, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-24T22:16:28,440 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:49026, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-24T22:16:28,440 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_2', locateType=CURRENT is [region=TestAcidGuarantees,,1732486585339.8ca644d2cada6dc44b363b7383dcd4fe., hostname=bd6592c37bb2,40087,1732486423713, seqNum=5] 2024-11-24T22:16:28,440 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_1', locateType=CURRENT is [region=TestAcidGuarantees,,1732486585339.8ca644d2cada6dc44b363b7383dcd4fe., hostname=bd6592c37bb2,40087,1732486423713, seqNum=5] 2024-11-24T22:16:28,441 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_0', locateType=CURRENT is [region=TestAcidGuarantees,,1732486585339.8ca644d2cada6dc44b363b7383dcd4fe., hostname=bd6592c37bb2,40087,1732486423713, seqNum=5] 2024-11-24T22:16:28,441 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37701 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=152 2024-11-24T22:16:28,442 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_0', locateType=CURRENT is [region=TestAcidGuarantees,,1732486585339.8ca644d2cada6dc44b363b7383dcd4fe., hostname=bd6592c37bb2,40087,1732486423713, seqNum=5] 2024-11-24T22:16:28,442 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_1', locateType=CURRENT is [region=TestAcidGuarantees,,1732486585339.8ca644d2cada6dc44b363b7383dcd4fe., hostname=bd6592c37bb2,40087,1732486423713, seqNum=5] 2024-11-24T22:16:28,444 DEBUG [Thread-2511 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-24T22:16:28,445 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=bd6592c37bb2,40087,1732486423713, seqNum=-1] 2024-11-24T22:16:28,446 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-24T22:16:28,447 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:49038, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-24T22:16:28,448 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_0', locateType=CURRENT is [region=TestAcidGuarantees,,1732486585339.8ca644d2cada6dc44b363b7383dcd4fe., hostname=bd6592c37bb2,40087,1732486423713, seqNum=5] 2024-11-24T22:16:28,449 DEBUG [Thread-2496 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-24T22:16:28,450 DEBUG [Thread-2496 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=bd6592c37bb2,40087,1732486423713, seqNum=-1] 2024-11-24T22:16:28,450 DEBUG [Thread-2496 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-24T22:16:28,451 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:49050, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-24T22:16:28,452 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_0', locateType=CURRENT is [region=TestAcidGuarantees,,1732486585339.8ca644d2cada6dc44b363b7383dcd4fe., hostname=bd6592c37bb2,40087,1732486423713, seqNum=5] 2024-11-24T22:16:28,453 DEBUG [Thread-2500 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-24T22:16:28,453 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40087 {}] regionserver.HRegion(8855): Flush requested on 8ca644d2cada6dc44b363b7383dcd4fe 2024-11-24T22:16:28,453 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 8ca644d2cada6dc44b363b7383dcd4fe 3/3 column families, dataSize=46.96 KB heapSize=140.20 KB 2024-11-24T22:16:28,453 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 8ca644d2cada6dc44b363b7383dcd4fe, store=A 2024-11-24T22:16:28,454 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-24T22:16:28,454 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 8ca644d2cada6dc44b363b7383dcd4fe, store=B 2024-11-24T22:16:28,454 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-24T22:16:28,454 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 8ca644d2cada6dc44b363b7383dcd4fe, store=C 2024-11-24T22:16:28,454 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-24T22:16:28,455 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=bd6592c37bb2,40087,1732486423713, seqNum=-1] 2024-11-24T22:16:28,455 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-24T22:16:28,456 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:49060, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-24T22:16:28,457 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_1', locateType=CURRENT is [region=TestAcidGuarantees,,1732486585339.8ca644d2cada6dc44b363b7383dcd4fe., hostname=bd6592c37bb2,40087,1732486423713, seqNum=5] 2024-11-24T22:16:28,483 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/bd6592c37bb2:0-2 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=309, cellSize=14145, totalHeapSize=32376, min timestamp=1732486588469, max timestamp=1732486588479Num uniques 859; 2024-11-24T22:16:28,483 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/bd6592c37bb2:0-2 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=14145, suffix heap size=117064,new segment heap size=32376 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=309 2024-11-24T22:16:28,483 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/bd6592c37bb2:0-3 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=309, cellSize=14145, totalHeapSize=32376, min timestamp=1732486588469, max timestamp=1732486588479Num uniques 859; 2024-11-24T22:16:28,483 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/bd6592c37bb2:0-3 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=14145, suffix heap size=117064,new segment heap size=32376 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=309 2024-11-24T22:16:28,484 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/bd6592c37bb2:0-5 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=209, cellSize=9565, totalHeapSize=21976, min timestamp=1732486588469, max timestamp=1732486588481Num uniques 859; 2024-11-24T22:16:28,484 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/bd6592c37bb2:0-5 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=9565, suffix heap size=117064,new segment heap size=21976 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=209 2024-11-24T22:16:28,489 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024112400a69d311b5b443f932c804300f5c47d_8ca644d2cada6dc44b363b7383dcd4fe is 50, key is test_row_0/A:col10/1732486588446/Put/seqid=0 2024-11-24T22:16:28,493 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45929 is added to blk_1073742386_1562 (size=12154) 2024-11-24T22:16:28,533 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40087 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8ca644d2cada6dc44b363b7383dcd4fe, server=bd6592c37bb2,40087,1732486423713 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-24T22:16:28,533 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40087 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49002 deadline: 1732486648525, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8ca644d2cada6dc44b363b7383dcd4fe, server=bd6592c37bb2,40087,1732486423713 2024-11-24T22:16:28,533 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40087 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8ca644d2cada6dc44b363b7383dcd4fe, server=bd6592c37bb2,40087,1732486423713 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-24T22:16:28,533 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40087 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8ca644d2cada6dc44b363b7383dcd4fe, server=bd6592c37bb2,40087,1732486423713 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-24T22:16:28,533 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40087 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48986 deadline: 1732486648527, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8ca644d2cada6dc44b363b7383dcd4fe, server=bd6592c37bb2,40087,1732486423713 2024-11-24T22:16:28,533 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732486585339.8ca644d2cada6dc44b363b7383dcd4fe., hostname=bd6592c37bb2,40087,1732486423713, seqNum=5 , the old value is region=TestAcidGuarantees,,1732486585339.8ca644d2cada6dc44b363b7383dcd4fe., hostname=bd6592c37bb2,40087,1732486423713, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8ca644d2cada6dc44b363b7383dcd4fe, server=bd6592c37bb2,40087,1732486423713 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T22:16:28,533 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40087 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49026 deadline: 1732486648527, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8ca644d2cada6dc44b363b7383dcd4fe, server=bd6592c37bb2,40087,1732486423713 2024-11-24T22:16:28,533 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732486585339.8ca644d2cada6dc44b363b7383dcd4fe., hostname=bd6592c37bb2,40087,1732486423713, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8ca644d2cada6dc44b363b7383dcd4fe, server=bd6592c37bb2,40087,1732486423713 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T22:16:28,533 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732486585339.8ca644d2cada6dc44b363b7383dcd4fe., hostname=bd6592c37bb2,40087,1732486423713, seqNum=5 because the exception is null or not the one we care about 2024-11-24T22:16:28,534 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732486585339.8ca644d2cada6dc44b363b7383dcd4fe., hostname=bd6592c37bb2,40087,1732486423713, seqNum=5 , the old value is region=TestAcidGuarantees,,1732486585339.8ca644d2cada6dc44b363b7383dcd4fe., hostname=bd6592c37bb2,40087,1732486423713, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8ca644d2cada6dc44b363b7383dcd4fe, server=bd6592c37bb2,40087,1732486423713 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T22:16:28,534 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732486585339.8ca644d2cada6dc44b363b7383dcd4fe., hostname=bd6592c37bb2,40087,1732486423713, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8ca644d2cada6dc44b363b7383dcd4fe, server=bd6592c37bb2,40087,1732486423713 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T22:16:28,534 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732486585339.8ca644d2cada6dc44b363b7383dcd4fe., hostname=bd6592c37bb2,40087,1732486423713, seqNum=5 because the exception is null or not the one we care about 2024-11-24T22:16:28,534 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732486585339.8ca644d2cada6dc44b363b7383dcd4fe., hostname=bd6592c37bb2,40087,1732486423713, seqNum=5 , the old value is region=TestAcidGuarantees,,1732486585339.8ca644d2cada6dc44b363b7383dcd4fe., hostname=bd6592c37bb2,40087,1732486423713, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8ca644d2cada6dc44b363b7383dcd4fe, server=bd6592c37bb2,40087,1732486423713 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T22:16:28,534 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732486585339.8ca644d2cada6dc44b363b7383dcd4fe., hostname=bd6592c37bb2,40087,1732486423713, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8ca644d2cada6dc44b363b7383dcd4fe, server=bd6592c37bb2,40087,1732486423713 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T22:16:28,534 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732486585339.8ca644d2cada6dc44b363b7383dcd4fe., hostname=bd6592c37bb2,40087,1732486423713, seqNum=5 because the exception is null or not the one we care about 2024-11-24T22:16:28,539 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40087 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8ca644d2cada6dc44b363b7383dcd4fe, server=bd6592c37bb2,40087,1732486423713 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-24T22:16:28,539 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40087 {}] ipc.CallRunner(138): callId: 7 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49060 deadline: 1732486648533, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8ca644d2cada6dc44b363b7383dcd4fe, server=bd6592c37bb2,40087,1732486423713 2024-11-24T22:16:28,539 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732486585339.8ca644d2cada6dc44b363b7383dcd4fe., hostname=bd6592c37bb2,40087,1732486423713, seqNum=5 , the old value is region=TestAcidGuarantees,,1732486585339.8ca644d2cada6dc44b363b7383dcd4fe., hostname=bd6592c37bb2,40087,1732486423713, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8ca644d2cada6dc44b363b7383dcd4fe, server=bd6592c37bb2,40087,1732486423713 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T22:16:28,539 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40087 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8ca644d2cada6dc44b363b7383dcd4fe, server=bd6592c37bb2,40087,1732486423713 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-24T22:16:28,539 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732486585339.8ca644d2cada6dc44b363b7383dcd4fe., hostname=bd6592c37bb2,40087,1732486423713, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8ca644d2cada6dc44b363b7383dcd4fe, server=bd6592c37bb2,40087,1732486423713 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T22:16:28,539 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732486585339.8ca644d2cada6dc44b363b7383dcd4fe., hostname=bd6592c37bb2,40087,1732486423713, seqNum=5 because the exception is null or not the one we care about 2024-11-24T22:16:28,539 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40087 {}] ipc.CallRunner(138): callId: 8 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49050 deadline: 1732486648533, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8ca644d2cada6dc44b363b7383dcd4fe, server=bd6592c37bb2,40087,1732486423713 2024-11-24T22:16:28,540 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732486585339.8ca644d2cada6dc44b363b7383dcd4fe., hostname=bd6592c37bb2,40087,1732486423713, seqNum=5 , the old value is region=TestAcidGuarantees,,1732486585339.8ca644d2cada6dc44b363b7383dcd4fe., hostname=bd6592c37bb2,40087,1732486423713, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8ca644d2cada6dc44b363b7383dcd4fe, server=bd6592c37bb2,40087,1732486423713 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T22:16:28,540 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732486585339.8ca644d2cada6dc44b363b7383dcd4fe., hostname=bd6592c37bb2,40087,1732486423713, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8ca644d2cada6dc44b363b7383dcd4fe, server=bd6592c37bb2,40087,1732486423713 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T22:16:28,540 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732486585339.8ca644d2cada6dc44b363b7383dcd4fe., hostname=bd6592c37bb2,40087,1732486423713, seqNum=5 because the exception is null or not the one we care about 2024-11-24T22:16:28,550 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37701 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=152 2024-11-24T22:16:28,592 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=40087 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=153 2024-11-24T22:16:28,592 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bd6592c37bb2:0-0 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732486585339.8ca644d2cada6dc44b363b7383dcd4fe. 2024-11-24T22:16:28,592 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bd6592c37bb2:0-0 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732486585339.8ca644d2cada6dc44b363b7383dcd4fe. as already flushing 2024-11-24T22:16:28,592 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bd6592c37bb2:0-0 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732486585339.8ca644d2cada6dc44b363b7383dcd4fe. 2024-11-24T22:16:28,592 ERROR [RS_FLUSH_OPERATIONS-regionserver/bd6592c37bb2:0-0 {event_type=RS_FLUSH_REGIONS, pid=153}] handler.RSProcedureHandler(58): pid=153 java.io.IOException: Unable to complete flush {ENCODED => 8ca644d2cada6dc44b363b7383dcd4fe, NAME => 'TestAcidGuarantees,,1732486585339.8ca644d2cada6dc44b363b7383dcd4fe.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T22:16:28,592 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bd6592c37bb2:0-0 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=153 java.io.IOException: Unable to complete flush {ENCODED => 8ca644d2cada6dc44b363b7383dcd4fe, NAME => 'TestAcidGuarantees,,1732486585339.8ca644d2cada6dc44b363b7383dcd4fe.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T22:16:28,593 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37701 {}] master.HMaster(4177): Remote procedure failed, pid=153 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 8ca644d2cada6dc44b363b7383dcd4fe, NAME => 'TestAcidGuarantees,,1732486585339.8ca644d2cada6dc44b363b7383dcd4fe.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 8ca644d2cada6dc44b363b7383dcd4fe, NAME => 'TestAcidGuarantees,,1732486585339.8ca644d2cada6dc44b363b7383dcd4fe.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T22:16:28,645 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40087 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8ca644d2cada6dc44b363b7383dcd4fe, server=bd6592c37bb2,40087,1732486423713 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-24T22:16:28,645 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40087 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48986 deadline: 1732486648641, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8ca644d2cada6dc44b363b7383dcd4fe, server=bd6592c37bb2,40087,1732486423713 2024-11-24T22:16:28,645 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40087 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8ca644d2cada6dc44b363b7383dcd4fe, server=bd6592c37bb2,40087,1732486423713 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-24T22:16:28,645 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732486585339.8ca644d2cada6dc44b363b7383dcd4fe., hostname=bd6592c37bb2,40087,1732486423713, seqNum=5 , the old value is region=TestAcidGuarantees,,1732486585339.8ca644d2cada6dc44b363b7383dcd4fe., hostname=bd6592c37bb2,40087,1732486423713, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8ca644d2cada6dc44b363b7383dcd4fe, server=bd6592c37bb2,40087,1732486423713 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T22:16:28,645 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40087 {}] ipc.CallRunner(138): callId: 11 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49002 deadline: 1732486648641, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8ca644d2cada6dc44b363b7383dcd4fe, server=bd6592c37bb2,40087,1732486423713 2024-11-24T22:16:28,645 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732486585339.8ca644d2cada6dc44b363b7383dcd4fe., hostname=bd6592c37bb2,40087,1732486423713, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8ca644d2cada6dc44b363b7383dcd4fe, server=bd6592c37bb2,40087,1732486423713 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T22:16:28,646 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732486585339.8ca644d2cada6dc44b363b7383dcd4fe., hostname=bd6592c37bb2,40087,1732486423713, seqNum=5 because the exception is null or not the one we care about 2024-11-24T22:16:28,646 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732486585339.8ca644d2cada6dc44b363b7383dcd4fe., hostname=bd6592c37bb2,40087,1732486423713, seqNum=5 , the old value is region=TestAcidGuarantees,,1732486585339.8ca644d2cada6dc44b363b7383dcd4fe., hostname=bd6592c37bb2,40087,1732486423713, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8ca644d2cada6dc44b363b7383dcd4fe, server=bd6592c37bb2,40087,1732486423713 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T22:16:28,646 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40087 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8ca644d2cada6dc44b363b7383dcd4fe, server=bd6592c37bb2,40087,1732486423713 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-24T22:16:28,646 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732486585339.8ca644d2cada6dc44b363b7383dcd4fe., hostname=bd6592c37bb2,40087,1732486423713, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8ca644d2cada6dc44b363b7383dcd4fe, server=bd6592c37bb2,40087,1732486423713 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T22:16:28,646 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732486585339.8ca644d2cada6dc44b363b7383dcd4fe., hostname=bd6592c37bb2,40087,1732486423713, seqNum=5 because the exception is null or not the one we care about 2024-11-24T22:16:28,646 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40087 {}] ipc.CallRunner(138): callId: 11 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49026 deadline: 1732486648641, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8ca644d2cada6dc44b363b7383dcd4fe, server=bd6592c37bb2,40087,1732486423713 2024-11-24T22:16:28,646 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732486585339.8ca644d2cada6dc44b363b7383dcd4fe., hostname=bd6592c37bb2,40087,1732486423713, seqNum=5 , the old value is region=TestAcidGuarantees,,1732486585339.8ca644d2cada6dc44b363b7383dcd4fe., hostname=bd6592c37bb2,40087,1732486423713, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8ca644d2cada6dc44b363b7383dcd4fe, server=bd6592c37bb2,40087,1732486423713 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T22:16:28,646 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732486585339.8ca644d2cada6dc44b363b7383dcd4fe., hostname=bd6592c37bb2,40087,1732486423713, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8ca644d2cada6dc44b363b7383dcd4fe, server=bd6592c37bb2,40087,1732486423713 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T22:16:28,646 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732486585339.8ca644d2cada6dc44b363b7383dcd4fe., hostname=bd6592c37bb2,40087,1732486423713, seqNum=5 because the exception is null or not the one we care about 2024-11-24T22:16:28,653 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40087 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8ca644d2cada6dc44b363b7383dcd4fe, server=bd6592c37bb2,40087,1732486423713 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-24T22:16:28,654 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40087 {}] ipc.CallRunner(138): callId: 8 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49060 deadline: 1732486648650, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8ca644d2cada6dc44b363b7383dcd4fe, server=bd6592c37bb2,40087,1732486423713 2024-11-24T22:16:28,654 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732486585339.8ca644d2cada6dc44b363b7383dcd4fe., hostname=bd6592c37bb2,40087,1732486423713, seqNum=5 , the old value is region=TestAcidGuarantees,,1732486585339.8ca644d2cada6dc44b363b7383dcd4fe., hostname=bd6592c37bb2,40087,1732486423713, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8ca644d2cada6dc44b363b7383dcd4fe, server=bd6592c37bb2,40087,1732486423713 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T22:16:28,654 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732486585339.8ca644d2cada6dc44b363b7383dcd4fe., hostname=bd6592c37bb2,40087,1732486423713, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8ca644d2cada6dc44b363b7383dcd4fe, server=bd6592c37bb2,40087,1732486423713 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T22:16:28,654 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40087 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8ca644d2cada6dc44b363b7383dcd4fe, server=bd6592c37bb2,40087,1732486423713 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-24T22:16:28,654 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732486585339.8ca644d2cada6dc44b363b7383dcd4fe., hostname=bd6592c37bb2,40087,1732486423713, seqNum=5 because the exception is null or not the one we care about 2024-11-24T22:16:28,654 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40087 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49050 deadline: 1732486648651, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8ca644d2cada6dc44b363b7383dcd4fe, server=bd6592c37bb2,40087,1732486423713 2024-11-24T22:16:28,654 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732486585339.8ca644d2cada6dc44b363b7383dcd4fe., hostname=bd6592c37bb2,40087,1732486423713, seqNum=5 , the old value is region=TestAcidGuarantees,,1732486585339.8ca644d2cada6dc44b363b7383dcd4fe., hostname=bd6592c37bb2,40087,1732486423713, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8ca644d2cada6dc44b363b7383dcd4fe, server=bd6592c37bb2,40087,1732486423713 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T22:16:28,654 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732486585339.8ca644d2cada6dc44b363b7383dcd4fe., hostname=bd6592c37bb2,40087,1732486423713, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8ca644d2cada6dc44b363b7383dcd4fe, server=bd6592c37bb2,40087,1732486423713 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T22:16:28,654 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732486585339.8ca644d2cada6dc44b363b7383dcd4fe., hostname=bd6592c37bb2,40087,1732486423713, seqNum=5 because the exception is null or not the one we care about 2024-11-24T22:16:28,711 WARN [HBase-Metrics2-1 {}] impl.MetricsConfig(138): Cannot locate configuration: tried hadoop-metrics2-hbase.properties,hadoop-metrics2.properties 2024-11-24T22:16:28,744 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=40087 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=153 2024-11-24T22:16:28,745 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bd6592c37bb2:0-1 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732486585339.8ca644d2cada6dc44b363b7383dcd4fe. 2024-11-24T22:16:28,745 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bd6592c37bb2:0-1 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732486585339.8ca644d2cada6dc44b363b7383dcd4fe. as already flushing 2024-11-24T22:16:28,745 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bd6592c37bb2:0-1 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732486585339.8ca644d2cada6dc44b363b7383dcd4fe. 2024-11-24T22:16:28,745 ERROR [RS_FLUSH_OPERATIONS-regionserver/bd6592c37bb2:0-1 {event_type=RS_FLUSH_REGIONS, pid=153}] handler.RSProcedureHandler(58): pid=153 java.io.IOException: Unable to complete flush {ENCODED => 8ca644d2cada6dc44b363b7383dcd4fe, NAME => 'TestAcidGuarantees,,1732486585339.8ca644d2cada6dc44b363b7383dcd4fe.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T22:16:28,745 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bd6592c37bb2:0-1 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=153 java.io.IOException: Unable to complete flush {ENCODED => 8ca644d2cada6dc44b363b7383dcd4fe, NAME => 'TestAcidGuarantees,,1732486585339.8ca644d2cada6dc44b363b7383dcd4fe.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T22:16:28,746 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37701 {}] master.HMaster(4177): Remote procedure failed, pid=153 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 8ca644d2cada6dc44b363b7383dcd4fe, NAME => 'TestAcidGuarantees,,1732486585339.8ca644d2cada6dc44b363b7383dcd4fe.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 8ca644d2cada6dc44b363b7383dcd4fe, NAME => 'TestAcidGuarantees,,1732486585339.8ca644d2cada6dc44b363b7383dcd4fe.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T22:16:28,760 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37701 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=152 2024-11-24T22:16:28,854 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40087 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8ca644d2cada6dc44b363b7383dcd4fe, server=bd6592c37bb2,40087,1732486423713 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-24T22:16:28,854 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40087 {}] ipc.CallRunner(138): callId: 11 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48986 deadline: 1732486648850, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8ca644d2cada6dc44b363b7383dcd4fe, server=bd6592c37bb2,40087,1732486423713 2024-11-24T22:16:28,854 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732486585339.8ca644d2cada6dc44b363b7383dcd4fe., hostname=bd6592c37bb2,40087,1732486423713, seqNum=5 , the old value is region=TestAcidGuarantees,,1732486585339.8ca644d2cada6dc44b363b7383dcd4fe., hostname=bd6592c37bb2,40087,1732486423713, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8ca644d2cada6dc44b363b7383dcd4fe, server=bd6592c37bb2,40087,1732486423713 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T22:16:28,854 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732486585339.8ca644d2cada6dc44b363b7383dcd4fe., hostname=bd6592c37bb2,40087,1732486423713, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8ca644d2cada6dc44b363b7383dcd4fe, server=bd6592c37bb2,40087,1732486423713 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T22:16:28,854 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732486585339.8ca644d2cada6dc44b363b7383dcd4fe., hostname=bd6592c37bb2,40087,1732486423713, seqNum=5 because the exception is null or not the one we care about 2024-11-24T22:16:28,854 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40087 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8ca644d2cada6dc44b363b7383dcd4fe, server=bd6592c37bb2,40087,1732486423713 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-24T22:16:28,854 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40087 {}] ipc.CallRunner(138): callId: 12 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49002 deadline: 1732486648850, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8ca644d2cada6dc44b363b7383dcd4fe, server=bd6592c37bb2,40087,1732486423713 2024-11-24T22:16:28,855 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732486585339.8ca644d2cada6dc44b363b7383dcd4fe., hostname=bd6592c37bb2,40087,1732486423713, seqNum=5 , the old value is region=TestAcidGuarantees,,1732486585339.8ca644d2cada6dc44b363b7383dcd4fe., hostname=bd6592c37bb2,40087,1732486423713, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8ca644d2cada6dc44b363b7383dcd4fe, server=bd6592c37bb2,40087,1732486423713 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T22:16:28,855 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732486585339.8ca644d2cada6dc44b363b7383dcd4fe., hostname=bd6592c37bb2,40087,1732486423713, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8ca644d2cada6dc44b363b7383dcd4fe, server=bd6592c37bb2,40087,1732486423713 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T22:16:28,855 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732486585339.8ca644d2cada6dc44b363b7383dcd4fe., hostname=bd6592c37bb2,40087,1732486423713, seqNum=5 because the exception is null or not the one we care about 2024-11-24T22:16:28,855 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40087 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8ca644d2cada6dc44b363b7383dcd4fe, server=bd6592c37bb2,40087,1732486423713 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-24T22:16:28,855 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40087 {}] ipc.CallRunner(138): callId: 12 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49026 deadline: 1732486648850, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8ca644d2cada6dc44b363b7383dcd4fe, server=bd6592c37bb2,40087,1732486423713 2024-11-24T22:16:28,855 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732486585339.8ca644d2cada6dc44b363b7383dcd4fe., hostname=bd6592c37bb2,40087,1732486423713, seqNum=5 , the old value is region=TestAcidGuarantees,,1732486585339.8ca644d2cada6dc44b363b7383dcd4fe., hostname=bd6592c37bb2,40087,1732486423713, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8ca644d2cada6dc44b363b7383dcd4fe, server=bd6592c37bb2,40087,1732486423713 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T22:16:28,855 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732486585339.8ca644d2cada6dc44b363b7383dcd4fe., hostname=bd6592c37bb2,40087,1732486423713, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8ca644d2cada6dc44b363b7383dcd4fe, server=bd6592c37bb2,40087,1732486423713 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T22:16:28,855 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732486585339.8ca644d2cada6dc44b363b7383dcd4fe., hostname=bd6592c37bb2,40087,1732486423713, seqNum=5 because the exception is null or not the one we care about 2024-11-24T22:16:28,865 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40087 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8ca644d2cada6dc44b363b7383dcd4fe, server=bd6592c37bb2,40087,1732486423713 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-24T22:16:28,865 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40087 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49060 deadline: 1732486648861, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8ca644d2cada6dc44b363b7383dcd4fe, server=bd6592c37bb2,40087,1732486423713 2024-11-24T22:16:28,865 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732486585339.8ca644d2cada6dc44b363b7383dcd4fe., hostname=bd6592c37bb2,40087,1732486423713, seqNum=5 , the old value is region=TestAcidGuarantees,,1732486585339.8ca644d2cada6dc44b363b7383dcd4fe., hostname=bd6592c37bb2,40087,1732486423713, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8ca644d2cada6dc44b363b7383dcd4fe, server=bd6592c37bb2,40087,1732486423713 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T22:16:28,865 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732486585339.8ca644d2cada6dc44b363b7383dcd4fe., hostname=bd6592c37bb2,40087,1732486423713, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8ca644d2cada6dc44b363b7383dcd4fe, server=bd6592c37bb2,40087,1732486423713 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T22:16:28,865 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732486585339.8ca644d2cada6dc44b363b7383dcd4fe., hostname=bd6592c37bb2,40087,1732486423713, seqNum=5 because the exception is null or not the one we care about 2024-11-24T22:16:28,865 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40087 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8ca644d2cada6dc44b363b7383dcd4fe, server=bd6592c37bb2,40087,1732486423713 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-24T22:16:28,866 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40087 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49050 deadline: 1732486648861, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8ca644d2cada6dc44b363b7383dcd4fe, server=bd6592c37bb2,40087,1732486423713 2024-11-24T22:16:28,866 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732486585339.8ca644d2cada6dc44b363b7383dcd4fe., hostname=bd6592c37bb2,40087,1732486423713, seqNum=5 , the old value is region=TestAcidGuarantees,,1732486585339.8ca644d2cada6dc44b363b7383dcd4fe., hostname=bd6592c37bb2,40087,1732486423713, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8ca644d2cada6dc44b363b7383dcd4fe, server=bd6592c37bb2,40087,1732486423713 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T22:16:28,866 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732486585339.8ca644d2cada6dc44b363b7383dcd4fe., hostname=bd6592c37bb2,40087,1732486423713, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8ca644d2cada6dc44b363b7383dcd4fe, server=bd6592c37bb2,40087,1732486423713 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T22:16:28,866 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732486585339.8ca644d2cada6dc44b363b7383dcd4fe., hostname=bd6592c37bb2,40087,1732486423713, seqNum=5 because the exception is null or not the one we care about 2024-11-24T22:16:28,894 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T22:16:28,897 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=40087 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=153 2024-11-24T22:16:28,897 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bd6592c37bb2:0-2 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732486585339.8ca644d2cada6dc44b363b7383dcd4fe. 2024-11-24T22:16:28,897 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024112400a69d311b5b443f932c804300f5c47d_8ca644d2cada6dc44b363b7383dcd4fe to hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024112400a69d311b5b443f932c804300f5c47d_8ca644d2cada6dc44b363b7383dcd4fe 2024-11-24T22:16:28,897 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bd6592c37bb2:0-2 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732486585339.8ca644d2cada6dc44b363b7383dcd4fe. as already flushing 2024-11-24T22:16:28,897 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bd6592c37bb2:0-2 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732486585339.8ca644d2cada6dc44b363b7383dcd4fe. 2024-11-24T22:16:28,897 ERROR [RS_FLUSH_OPERATIONS-regionserver/bd6592c37bb2:0-2 {event_type=RS_FLUSH_REGIONS, pid=153}] handler.RSProcedureHandler(58): pid=153 java.io.IOException: Unable to complete flush {ENCODED => 8ca644d2cada6dc44b363b7383dcd4fe, NAME => 'TestAcidGuarantees,,1732486585339.8ca644d2cada6dc44b363b7383dcd4fe.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T22:16:28,898 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bd6592c37bb2:0-2 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=153 java.io.IOException: Unable to complete flush {ENCODED => 8ca644d2cada6dc44b363b7383dcd4fe, NAME => 'TestAcidGuarantees,,1732486585339.8ca644d2cada6dc44b363b7383dcd4fe.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T22:16:28,898 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/8ca644d2cada6dc44b363b7383dcd4fe/.tmp/A/4d8ef082bbc741109521c9a43715f049, store: [table=TestAcidGuarantees family=A region=8ca644d2cada6dc44b363b7383dcd4fe] 2024-11-24T22:16:28,898 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37701 {}] master.HMaster(4177): Remote procedure failed, pid=153 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 8ca644d2cada6dc44b363b7383dcd4fe, NAME => 'TestAcidGuarantees,,1732486585339.8ca644d2cada6dc44b363b7383dcd4fe.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 8ca644d2cada6dc44b363b7383dcd4fe, NAME => 'TestAcidGuarantees,,1732486585339.8ca644d2cada6dc44b363b7383dcd4fe.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T22:16:28,899 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/8ca644d2cada6dc44b363b7383dcd4fe/.tmp/A/4d8ef082bbc741109521c9a43715f049 is 175, key is test_row_0/A:col10/1732486588446/Put/seqid=0 2024-11-24T22:16:28,902 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45929 is added to blk_1073742387_1563 (size=30955) 2024-11-24T22:16:29,050 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=40087 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=153 2024-11-24T22:16:29,050 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bd6592c37bb2:0-0 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732486585339.8ca644d2cada6dc44b363b7383dcd4fe. 2024-11-24T22:16:29,050 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bd6592c37bb2:0-0 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732486585339.8ca644d2cada6dc44b363b7383dcd4fe. as already flushing 2024-11-24T22:16:29,050 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bd6592c37bb2:0-0 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732486585339.8ca644d2cada6dc44b363b7383dcd4fe. 2024-11-24T22:16:29,050 ERROR [RS_FLUSH_OPERATIONS-regionserver/bd6592c37bb2:0-0 {event_type=RS_FLUSH_REGIONS, pid=153}] handler.RSProcedureHandler(58): pid=153 java.io.IOException: Unable to complete flush {ENCODED => 8ca644d2cada6dc44b363b7383dcd4fe, NAME => 'TestAcidGuarantees,,1732486585339.8ca644d2cada6dc44b363b7383dcd4fe.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T22:16:29,050 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bd6592c37bb2:0-0 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=153 java.io.IOException: Unable to complete flush {ENCODED => 8ca644d2cada6dc44b363b7383dcd4fe, NAME => 'TestAcidGuarantees,,1732486585339.8ca644d2cada6dc44b363b7383dcd4fe.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T22:16:29,051 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37701 {}] master.HMaster(4177): Remote procedure failed, pid=153 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 8ca644d2cada6dc44b363b7383dcd4fe, NAME => 'TestAcidGuarantees,,1732486585339.8ca644d2cada6dc44b363b7383dcd4fe.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 8ca644d2cada6dc44b363b7383dcd4fe, NAME => 'TestAcidGuarantees,,1732486585339.8ca644d2cada6dc44b363b7383dcd4fe.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T22:16:29,070 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37701 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=152 2024-11-24T22:16:29,163 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40087 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8ca644d2cada6dc44b363b7383dcd4fe, server=bd6592c37bb2,40087,1732486423713 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-24T22:16:29,164 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40087 {}] ipc.CallRunner(138): callId: 12 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48986 deadline: 1732486649160, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8ca644d2cada6dc44b363b7383dcd4fe, server=bd6592c37bb2,40087,1732486423713 2024-11-24T22:16:29,164 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40087 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8ca644d2cada6dc44b363b7383dcd4fe, server=bd6592c37bb2,40087,1732486423713 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-24T22:16:29,164 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732486585339.8ca644d2cada6dc44b363b7383dcd4fe., hostname=bd6592c37bb2,40087,1732486423713, seqNum=5 , the old value is region=TestAcidGuarantees,,1732486585339.8ca644d2cada6dc44b363b7383dcd4fe., hostname=bd6592c37bb2,40087,1732486423713, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8ca644d2cada6dc44b363b7383dcd4fe, server=bd6592c37bb2,40087,1732486423713 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T22:16:29,164 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40087 {}] ipc.CallRunner(138): callId: 13 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49002 deadline: 1732486649161, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8ca644d2cada6dc44b363b7383dcd4fe, server=bd6592c37bb2,40087,1732486423713 2024-11-24T22:16:29,164 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732486585339.8ca644d2cada6dc44b363b7383dcd4fe., hostname=bd6592c37bb2,40087,1732486423713, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8ca644d2cada6dc44b363b7383dcd4fe, server=bd6592c37bb2,40087,1732486423713 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T22:16:29,164 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732486585339.8ca644d2cada6dc44b363b7383dcd4fe., hostname=bd6592c37bb2,40087,1732486423713, seqNum=5 because the exception is null or not the one we care about 2024-11-24T22:16:29,164 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40087 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8ca644d2cada6dc44b363b7383dcd4fe, server=bd6592c37bb2,40087,1732486423713 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-24T22:16:29,164 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732486585339.8ca644d2cada6dc44b363b7383dcd4fe., hostname=bd6592c37bb2,40087,1732486423713, seqNum=5 , the old value is region=TestAcidGuarantees,,1732486585339.8ca644d2cada6dc44b363b7383dcd4fe., hostname=bd6592c37bb2,40087,1732486423713, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8ca644d2cada6dc44b363b7383dcd4fe, server=bd6592c37bb2,40087,1732486423713 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T22:16:29,164 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732486585339.8ca644d2cada6dc44b363b7383dcd4fe., hostname=bd6592c37bb2,40087,1732486423713, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8ca644d2cada6dc44b363b7383dcd4fe, server=bd6592c37bb2,40087,1732486423713 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T22:16:29,164 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40087 {}] ipc.CallRunner(138): callId: 13 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49026 deadline: 1732486649161, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8ca644d2cada6dc44b363b7383dcd4fe, server=bd6592c37bb2,40087,1732486423713 2024-11-24T22:16:29,164 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732486585339.8ca644d2cada6dc44b363b7383dcd4fe., hostname=bd6592c37bb2,40087,1732486423713, seqNum=5 because the exception is null or not the one we care about 2024-11-24T22:16:29,164 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732486585339.8ca644d2cada6dc44b363b7383dcd4fe., hostname=bd6592c37bb2,40087,1732486423713, seqNum=5 , the old value is region=TestAcidGuarantees,,1732486585339.8ca644d2cada6dc44b363b7383dcd4fe., hostname=bd6592c37bb2,40087,1732486423713, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8ca644d2cada6dc44b363b7383dcd4fe, server=bd6592c37bb2,40087,1732486423713 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T22:16:29,164 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732486585339.8ca644d2cada6dc44b363b7383dcd4fe., hostname=bd6592c37bb2,40087,1732486423713, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8ca644d2cada6dc44b363b7383dcd4fe, server=bd6592c37bb2,40087,1732486423713 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T22:16:29,164 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732486585339.8ca644d2cada6dc44b363b7383dcd4fe., hostname=bd6592c37bb2,40087,1732486423713, seqNum=5 because the exception is null or not the one we care about 2024-11-24T22:16:29,174 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40087 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8ca644d2cada6dc44b363b7383dcd4fe, server=bd6592c37bb2,40087,1732486423713 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-24T22:16:29,174 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40087 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49060 deadline: 1732486649171, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8ca644d2cada6dc44b363b7383dcd4fe, server=bd6592c37bb2,40087,1732486423713 2024-11-24T22:16:29,175 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732486585339.8ca644d2cada6dc44b363b7383dcd4fe., hostname=bd6592c37bb2,40087,1732486423713, seqNum=5 , the old value is region=TestAcidGuarantees,,1732486585339.8ca644d2cada6dc44b363b7383dcd4fe., hostname=bd6592c37bb2,40087,1732486423713, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8ca644d2cada6dc44b363b7383dcd4fe, server=bd6592c37bb2,40087,1732486423713 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T22:16:29,175 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732486585339.8ca644d2cada6dc44b363b7383dcd4fe., hostname=bd6592c37bb2,40087,1732486423713, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8ca644d2cada6dc44b363b7383dcd4fe, server=bd6592c37bb2,40087,1732486423713 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T22:16:29,175 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732486585339.8ca644d2cada6dc44b363b7383dcd4fe., hostname=bd6592c37bb2,40087,1732486423713, seqNum=5 because the exception is null or not the one we care about 2024-11-24T22:16:29,175 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40087 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8ca644d2cada6dc44b363b7383dcd4fe, server=bd6592c37bb2,40087,1732486423713 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-24T22:16:29,175 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40087 {}] ipc.CallRunner(138): callId: 11 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49050 deadline: 1732486649171, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8ca644d2cada6dc44b363b7383dcd4fe, server=bd6592c37bb2,40087,1732486423713 2024-11-24T22:16:29,176 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732486585339.8ca644d2cada6dc44b363b7383dcd4fe., hostname=bd6592c37bb2,40087,1732486423713, seqNum=5 , the old value is region=TestAcidGuarantees,,1732486585339.8ca644d2cada6dc44b363b7383dcd4fe., hostname=bd6592c37bb2,40087,1732486423713, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8ca644d2cada6dc44b363b7383dcd4fe, server=bd6592c37bb2,40087,1732486423713 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T22:16:29,176 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732486585339.8ca644d2cada6dc44b363b7383dcd4fe., hostname=bd6592c37bb2,40087,1732486423713, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8ca644d2cada6dc44b363b7383dcd4fe, server=bd6592c37bb2,40087,1732486423713 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T22:16:29,176 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732486585339.8ca644d2cada6dc44b363b7383dcd4fe., hostname=bd6592c37bb2,40087,1732486423713, seqNum=5 because the exception is null or not the one we care about 2024-11-24T22:16:29,203 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=40087 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=153 2024-11-24T22:16:29,203 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bd6592c37bb2:0-1 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732486585339.8ca644d2cada6dc44b363b7383dcd4fe. 2024-11-24T22:16:29,203 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bd6592c37bb2:0-1 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732486585339.8ca644d2cada6dc44b363b7383dcd4fe. as already flushing 2024-11-24T22:16:29,203 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bd6592c37bb2:0-1 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732486585339.8ca644d2cada6dc44b363b7383dcd4fe. 2024-11-24T22:16:29,203 ERROR [RS_FLUSH_OPERATIONS-regionserver/bd6592c37bb2:0-1 {event_type=RS_FLUSH_REGIONS, pid=153}] handler.RSProcedureHandler(58): pid=153 java.io.IOException: Unable to complete flush {ENCODED => 8ca644d2cada6dc44b363b7383dcd4fe, NAME => 'TestAcidGuarantees,,1732486585339.8ca644d2cada6dc44b363b7383dcd4fe.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T22:16:29,203 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bd6592c37bb2:0-1 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=153 java.io.IOException: Unable to complete flush {ENCODED => 8ca644d2cada6dc44b363b7383dcd4fe, NAME => 'TestAcidGuarantees,,1732486585339.8ca644d2cada6dc44b363b7383dcd4fe.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T22:16:29,204 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37701 {}] master.HMaster(4177): Remote procedure failed, pid=153 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 8ca644d2cada6dc44b363b7383dcd4fe, NAME => 'TestAcidGuarantees,,1732486585339.8ca644d2cada6dc44b363b7383dcd4fe.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 8ca644d2cada6dc44b363b7383dcd4fe, NAME => 'TestAcidGuarantees,,1732486585339.8ca644d2cada6dc44b363b7383dcd4fe.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T22:16:29,302 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=14, memsize=15.7 K, hasBloomFilter=true, into tmp file hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/8ca644d2cada6dc44b363b7383dcd4fe/.tmp/A/4d8ef082bbc741109521c9a43715f049 2024-11-24T22:16:29,326 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/8ca644d2cada6dc44b363b7383dcd4fe/.tmp/B/be99041ff522499f83a39a00787ed3ce is 50, key is test_row_0/B:col10/1732486588446/Put/seqid=0 2024-11-24T22:16:29,329 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45929 is added to blk_1073742388_1564 (size=12001) 2024-11-24T22:16:29,355 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=40087 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=153 2024-11-24T22:16:29,355 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bd6592c37bb2:0-2 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732486585339.8ca644d2cada6dc44b363b7383dcd4fe. 2024-11-24T22:16:29,355 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bd6592c37bb2:0-2 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732486585339.8ca644d2cada6dc44b363b7383dcd4fe. as already flushing 2024-11-24T22:16:29,355 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bd6592c37bb2:0-2 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732486585339.8ca644d2cada6dc44b363b7383dcd4fe. 2024-11-24T22:16:29,356 ERROR [RS_FLUSH_OPERATIONS-regionserver/bd6592c37bb2:0-2 {event_type=RS_FLUSH_REGIONS, pid=153}] handler.RSProcedureHandler(58): pid=153 java.io.IOException: Unable to complete flush {ENCODED => 8ca644d2cada6dc44b363b7383dcd4fe, NAME => 'TestAcidGuarantees,,1732486585339.8ca644d2cada6dc44b363b7383dcd4fe.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T22:16:29,356 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bd6592c37bb2:0-2 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=153 java.io.IOException: Unable to complete flush {ENCODED => 8ca644d2cada6dc44b363b7383dcd4fe, NAME => 'TestAcidGuarantees,,1732486585339.8ca644d2cada6dc44b363b7383dcd4fe.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T22:16:29,356 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37701 {}] master.HMaster(4177): Remote procedure failed, pid=153 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 8ca644d2cada6dc44b363b7383dcd4fe, NAME => 'TestAcidGuarantees,,1732486585339.8ca644d2cada6dc44b363b7383dcd4fe.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 8ca644d2cada6dc44b363b7383dcd4fe, NAME => 'TestAcidGuarantees,,1732486585339.8ca644d2cada6dc44b363b7383dcd4fe.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T22:16:29,508 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=40087 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=153 2024-11-24T22:16:29,508 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bd6592c37bb2:0-0 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732486585339.8ca644d2cada6dc44b363b7383dcd4fe. 2024-11-24T22:16:29,508 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bd6592c37bb2:0-0 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732486585339.8ca644d2cada6dc44b363b7383dcd4fe. as already flushing 2024-11-24T22:16:29,508 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bd6592c37bb2:0-0 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732486585339.8ca644d2cada6dc44b363b7383dcd4fe. 2024-11-24T22:16:29,508 ERROR [RS_FLUSH_OPERATIONS-regionserver/bd6592c37bb2:0-0 {event_type=RS_FLUSH_REGIONS, pid=153}] handler.RSProcedureHandler(58): pid=153 java.io.IOException: Unable to complete flush {ENCODED => 8ca644d2cada6dc44b363b7383dcd4fe, NAME => 'TestAcidGuarantees,,1732486585339.8ca644d2cada6dc44b363b7383dcd4fe.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T22:16:29,508 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bd6592c37bb2:0-0 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=153 java.io.IOException: Unable to complete flush {ENCODED => 8ca644d2cada6dc44b363b7383dcd4fe, NAME => 'TestAcidGuarantees,,1732486585339.8ca644d2cada6dc44b363b7383dcd4fe.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T22:16:29,509 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37701 {}] master.HMaster(4177): Remote procedure failed, pid=153 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 8ca644d2cada6dc44b363b7383dcd4fe, NAME => 'TestAcidGuarantees,,1732486585339.8ca644d2cada6dc44b363b7383dcd4fe.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 8ca644d2cada6dc44b363b7383dcd4fe, NAME => 'TestAcidGuarantees,,1732486585339.8ca644d2cada6dc44b363b7383dcd4fe.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T22:16:29,581 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37701 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=152 2024-11-24T22:16:29,660 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=40087 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=153 2024-11-24T22:16:29,661 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bd6592c37bb2:0-1 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732486585339.8ca644d2cada6dc44b363b7383dcd4fe. 2024-11-24T22:16:29,661 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bd6592c37bb2:0-1 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732486585339.8ca644d2cada6dc44b363b7383dcd4fe. as already flushing 2024-11-24T22:16:29,661 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bd6592c37bb2:0-1 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732486585339.8ca644d2cada6dc44b363b7383dcd4fe. 2024-11-24T22:16:29,661 ERROR [RS_FLUSH_OPERATIONS-regionserver/bd6592c37bb2:0-1 {event_type=RS_FLUSH_REGIONS, pid=153}] handler.RSProcedureHandler(58): pid=153 java.io.IOException: Unable to complete flush {ENCODED => 8ca644d2cada6dc44b363b7383dcd4fe, NAME => 'TestAcidGuarantees,,1732486585339.8ca644d2cada6dc44b363b7383dcd4fe.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T22:16:29,661 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bd6592c37bb2:0-1 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=153 java.io.IOException: Unable to complete flush {ENCODED => 8ca644d2cada6dc44b363b7383dcd4fe, NAME => 'TestAcidGuarantees,,1732486585339.8ca644d2cada6dc44b363b7383dcd4fe.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T22:16:29,662 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37701 {}] master.HMaster(4177): Remote procedure failed, pid=153 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 8ca644d2cada6dc44b363b7383dcd4fe, NAME => 'TestAcidGuarantees,,1732486585339.8ca644d2cada6dc44b363b7383dcd4fe.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 8ca644d2cada6dc44b363b7383dcd4fe, NAME => 'TestAcidGuarantees,,1732486585339.8ca644d2cada6dc44b363b7383dcd4fe.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T22:16:29,674 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40087 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8ca644d2cada6dc44b363b7383dcd4fe, server=bd6592c37bb2,40087,1732486423713 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-24T22:16:29,674 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40087 {}] ipc.CallRunner(138): callId: 13 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48986 deadline: 1732486649671, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8ca644d2cada6dc44b363b7383dcd4fe, server=bd6592c37bb2,40087,1732486423713 2024-11-24T22:16:29,675 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40087 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8ca644d2cada6dc44b363b7383dcd4fe, server=bd6592c37bb2,40087,1732486423713 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-24T22:16:29,675 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40087 {}] ipc.CallRunner(138): callId: 14 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49002 deadline: 1732486649671, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8ca644d2cada6dc44b363b7383dcd4fe, server=bd6592c37bb2,40087,1732486423713 2024-11-24T22:16:29,675 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732486585339.8ca644d2cada6dc44b363b7383dcd4fe., hostname=bd6592c37bb2,40087,1732486423713, seqNum=5 , the old value is region=TestAcidGuarantees,,1732486585339.8ca644d2cada6dc44b363b7383dcd4fe., hostname=bd6592c37bb2,40087,1732486423713, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8ca644d2cada6dc44b363b7383dcd4fe, server=bd6592c37bb2,40087,1732486423713 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T22:16:29,675 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732486585339.8ca644d2cada6dc44b363b7383dcd4fe., hostname=bd6592c37bb2,40087,1732486423713, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8ca644d2cada6dc44b363b7383dcd4fe, server=bd6592c37bb2,40087,1732486423713 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T22:16:29,675 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732486585339.8ca644d2cada6dc44b363b7383dcd4fe., hostname=bd6592c37bb2,40087,1732486423713, seqNum=5 because the exception is null or not the one we care about 2024-11-24T22:16:29,675 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732486585339.8ca644d2cada6dc44b363b7383dcd4fe., hostname=bd6592c37bb2,40087,1732486423713, seqNum=5 , the old value is region=TestAcidGuarantees,,1732486585339.8ca644d2cada6dc44b363b7383dcd4fe., hostname=bd6592c37bb2,40087,1732486423713, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8ca644d2cada6dc44b363b7383dcd4fe, server=bd6592c37bb2,40087,1732486423713 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T22:16:29,675 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732486585339.8ca644d2cada6dc44b363b7383dcd4fe., hostname=bd6592c37bb2,40087,1732486423713, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8ca644d2cada6dc44b363b7383dcd4fe, server=bd6592c37bb2,40087,1732486423713 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T22:16:29,675 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40087 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8ca644d2cada6dc44b363b7383dcd4fe, server=bd6592c37bb2,40087,1732486423713 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-24T22:16:29,675 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732486585339.8ca644d2cada6dc44b363b7383dcd4fe., hostname=bd6592c37bb2,40087,1732486423713, seqNum=5 because the exception is null or not the one we care about 2024-11-24T22:16:29,675 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40087 {}] ipc.CallRunner(138): callId: 14 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49026 deadline: 1732486649671, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8ca644d2cada6dc44b363b7383dcd4fe, server=bd6592c37bb2,40087,1732486423713 2024-11-24T22:16:29,675 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732486585339.8ca644d2cada6dc44b363b7383dcd4fe., hostname=bd6592c37bb2,40087,1732486423713, seqNum=5 , the old value is region=TestAcidGuarantees,,1732486585339.8ca644d2cada6dc44b363b7383dcd4fe., hostname=bd6592c37bb2,40087,1732486423713, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8ca644d2cada6dc44b363b7383dcd4fe, server=bd6592c37bb2,40087,1732486423713 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T22:16:29,675 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732486585339.8ca644d2cada6dc44b363b7383dcd4fe., hostname=bd6592c37bb2,40087,1732486423713, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8ca644d2cada6dc44b363b7383dcd4fe, server=bd6592c37bb2,40087,1732486423713 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T22:16:29,675 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732486585339.8ca644d2cada6dc44b363b7383dcd4fe., hostname=bd6592c37bb2,40087,1732486423713, seqNum=5 because the exception is null or not the one we care about 2024-11-24T22:16:29,683 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40087 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8ca644d2cada6dc44b363b7383dcd4fe, server=bd6592c37bb2,40087,1732486423713 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-24T22:16:29,683 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40087 {}] ipc.CallRunner(138): callId: 11 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49060 deadline: 1732486649680, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8ca644d2cada6dc44b363b7383dcd4fe, server=bd6592c37bb2,40087,1732486423713 2024-11-24T22:16:29,683 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40087 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8ca644d2cada6dc44b363b7383dcd4fe, server=bd6592c37bb2,40087,1732486423713 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-24T22:16:29,683 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40087 {}] ipc.CallRunner(138): callId: 12 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49050 deadline: 1732486649680, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8ca644d2cada6dc44b363b7383dcd4fe, server=bd6592c37bb2,40087,1732486423713 2024-11-24T22:16:29,683 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732486585339.8ca644d2cada6dc44b363b7383dcd4fe., hostname=bd6592c37bb2,40087,1732486423713, seqNum=5 , the old value is region=TestAcidGuarantees,,1732486585339.8ca644d2cada6dc44b363b7383dcd4fe., hostname=bd6592c37bb2,40087,1732486423713, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8ca644d2cada6dc44b363b7383dcd4fe, server=bd6592c37bb2,40087,1732486423713 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T22:16:29,684 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732486585339.8ca644d2cada6dc44b363b7383dcd4fe., hostname=bd6592c37bb2,40087,1732486423713, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8ca644d2cada6dc44b363b7383dcd4fe, server=bd6592c37bb2,40087,1732486423713 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T22:16:29,684 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732486585339.8ca644d2cada6dc44b363b7383dcd4fe., hostname=bd6592c37bb2,40087,1732486423713, seqNum=5 because the exception is null or not the one we care about 2024-11-24T22:16:29,684 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732486585339.8ca644d2cada6dc44b363b7383dcd4fe., hostname=bd6592c37bb2,40087,1732486423713, seqNum=5 , the old value is region=TestAcidGuarantees,,1732486585339.8ca644d2cada6dc44b363b7383dcd4fe., hostname=bd6592c37bb2,40087,1732486423713, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8ca644d2cada6dc44b363b7383dcd4fe, server=bd6592c37bb2,40087,1732486423713 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T22:16:29,684 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732486585339.8ca644d2cada6dc44b363b7383dcd4fe., hostname=bd6592c37bb2,40087,1732486423713, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8ca644d2cada6dc44b363b7383dcd4fe, server=bd6592c37bb2,40087,1732486423713 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T22:16:29,684 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732486585339.8ca644d2cada6dc44b363b7383dcd4fe., hostname=bd6592c37bb2,40087,1732486423713, seqNum=5 because the exception is null or not the one we care about 2024-11-24T22:16:29,730 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=14 (bloomFilter=true), to=hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/8ca644d2cada6dc44b363b7383dcd4fe/.tmp/B/be99041ff522499f83a39a00787ed3ce 2024-11-24T22:16:29,753 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/8ca644d2cada6dc44b363b7383dcd4fe/.tmp/C/073e5ad8128f4a9fa041c49b3dd915d6 is 50, key is test_row_0/C:col10/1732486588446/Put/seqid=0 2024-11-24T22:16:29,756 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45929 is added to blk_1073742389_1565 (size=12001) 2024-11-24T22:16:29,813 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=40087 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=153 2024-11-24T22:16:29,813 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bd6592c37bb2:0-2 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732486585339.8ca644d2cada6dc44b363b7383dcd4fe. 2024-11-24T22:16:29,813 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bd6592c37bb2:0-2 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732486585339.8ca644d2cada6dc44b363b7383dcd4fe. as already flushing 2024-11-24T22:16:29,813 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bd6592c37bb2:0-2 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732486585339.8ca644d2cada6dc44b363b7383dcd4fe. 2024-11-24T22:16:29,813 ERROR [RS_FLUSH_OPERATIONS-regionserver/bd6592c37bb2:0-2 {event_type=RS_FLUSH_REGIONS, pid=153}] handler.RSProcedureHandler(58): pid=153 java.io.IOException: Unable to complete flush {ENCODED => 8ca644d2cada6dc44b363b7383dcd4fe, NAME => 'TestAcidGuarantees,,1732486585339.8ca644d2cada6dc44b363b7383dcd4fe.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T22:16:29,814 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bd6592c37bb2:0-2 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=153 java.io.IOException: Unable to complete flush {ENCODED => 8ca644d2cada6dc44b363b7383dcd4fe, NAME => 'TestAcidGuarantees,,1732486585339.8ca644d2cada6dc44b363b7383dcd4fe.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T22:16:29,814 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37701 {}] master.HMaster(4177): Remote procedure failed, pid=153 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 8ca644d2cada6dc44b363b7383dcd4fe, NAME => 'TestAcidGuarantees,,1732486585339.8ca644d2cada6dc44b363b7383dcd4fe.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 8ca644d2cada6dc44b363b7383dcd4fe, NAME => 'TestAcidGuarantees,,1732486585339.8ca644d2cada6dc44b363b7383dcd4fe.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T22:16:29,966 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=40087 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=153 2024-11-24T22:16:29,966 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bd6592c37bb2:0-0 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732486585339.8ca644d2cada6dc44b363b7383dcd4fe. 2024-11-24T22:16:29,966 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bd6592c37bb2:0-0 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732486585339.8ca644d2cada6dc44b363b7383dcd4fe. as already flushing 2024-11-24T22:16:29,966 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bd6592c37bb2:0-0 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732486585339.8ca644d2cada6dc44b363b7383dcd4fe. 2024-11-24T22:16:29,966 ERROR [RS_FLUSH_OPERATIONS-regionserver/bd6592c37bb2:0-0 {event_type=RS_FLUSH_REGIONS, pid=153}] handler.RSProcedureHandler(58): pid=153 java.io.IOException: Unable to complete flush {ENCODED => 8ca644d2cada6dc44b363b7383dcd4fe, NAME => 'TestAcidGuarantees,,1732486585339.8ca644d2cada6dc44b363b7383dcd4fe.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T22:16:29,966 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bd6592c37bb2:0-0 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=153 java.io.IOException: Unable to complete flush {ENCODED => 8ca644d2cada6dc44b363b7383dcd4fe, NAME => 'TestAcidGuarantees,,1732486585339.8ca644d2cada6dc44b363b7383dcd4fe.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T22:16:29,967 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37701 {}] master.HMaster(4177): Remote procedure failed, pid=153 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 8ca644d2cada6dc44b363b7383dcd4fe, NAME => 'TestAcidGuarantees,,1732486585339.8ca644d2cada6dc44b363b7383dcd4fe.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 8ca644d2cada6dc44b363b7383dcd4fe, NAME => 'TestAcidGuarantees,,1732486585339.8ca644d2cada6dc44b363b7383dcd4fe.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T22:16:30,118 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=40087 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=153 2024-11-24T22:16:30,119 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bd6592c37bb2:0-1 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732486585339.8ca644d2cada6dc44b363b7383dcd4fe. 2024-11-24T22:16:30,119 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bd6592c37bb2:0-1 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732486585339.8ca644d2cada6dc44b363b7383dcd4fe. as already flushing 2024-11-24T22:16:30,119 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bd6592c37bb2:0-1 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732486585339.8ca644d2cada6dc44b363b7383dcd4fe. 2024-11-24T22:16:30,119 ERROR [RS_FLUSH_OPERATIONS-regionserver/bd6592c37bb2:0-1 {event_type=RS_FLUSH_REGIONS, pid=153}] handler.RSProcedureHandler(58): pid=153 java.io.IOException: Unable to complete flush {ENCODED => 8ca644d2cada6dc44b363b7383dcd4fe, NAME => 'TestAcidGuarantees,,1732486585339.8ca644d2cada6dc44b363b7383dcd4fe.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T22:16:30,119 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bd6592c37bb2:0-1 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=153 java.io.IOException: Unable to complete flush {ENCODED => 8ca644d2cada6dc44b363b7383dcd4fe, NAME => 'TestAcidGuarantees,,1732486585339.8ca644d2cada6dc44b363b7383dcd4fe.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T22:16:30,119 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37701 {}] master.HMaster(4177): Remote procedure failed, pid=153 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 8ca644d2cada6dc44b363b7383dcd4fe, NAME => 'TestAcidGuarantees,,1732486585339.8ca644d2cada6dc44b363b7383dcd4fe.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 8ca644d2cada6dc44b363b7383dcd4fe, NAME => 'TestAcidGuarantees,,1732486585339.8ca644d2cada6dc44b363b7383dcd4fe.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T22:16:30,157 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=14 (bloomFilter=true), to=hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/8ca644d2cada6dc44b363b7383dcd4fe/.tmp/C/073e5ad8128f4a9fa041c49b3dd915d6 2024-11-24T22:16:30,161 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/8ca644d2cada6dc44b363b7383dcd4fe/.tmp/A/4d8ef082bbc741109521c9a43715f049 as hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/8ca644d2cada6dc44b363b7383dcd4fe/A/4d8ef082bbc741109521c9a43715f049 2024-11-24T22:16:30,164 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/8ca644d2cada6dc44b363b7383dcd4fe/A/4d8ef082bbc741109521c9a43715f049, entries=150, sequenceid=14, filesize=30.2 K 2024-11-24T22:16:30,165 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/8ca644d2cada6dc44b363b7383dcd4fe/.tmp/B/be99041ff522499f83a39a00787ed3ce as hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/8ca644d2cada6dc44b363b7383dcd4fe/B/be99041ff522499f83a39a00787ed3ce 2024-11-24T22:16:30,168 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/8ca644d2cada6dc44b363b7383dcd4fe/B/be99041ff522499f83a39a00787ed3ce, entries=150, sequenceid=14, filesize=11.7 K 2024-11-24T22:16:30,168 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/8ca644d2cada6dc44b363b7383dcd4fe/.tmp/C/073e5ad8128f4a9fa041c49b3dd915d6 as hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/8ca644d2cada6dc44b363b7383dcd4fe/C/073e5ad8128f4a9fa041c49b3dd915d6 2024-11-24T22:16:30,171 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/8ca644d2cada6dc44b363b7383dcd4fe/C/073e5ad8128f4a9fa041c49b3dd915d6, entries=150, sequenceid=14, filesize=11.7 K 2024-11-24T22:16:30,172 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~46.96 KB/48090, heapSize ~140.16 KB/143520, currentSize=136.42 KB/139690 for 8ca644d2cada6dc44b363b7383dcd4fe in 1719ms, sequenceid=14, compaction requested=false 2024-11-24T22:16:30,172 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 8ca644d2cada6dc44b363b7383dcd4fe: 2024-11-24T22:16:30,271 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=40087 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=153 2024-11-24T22:16:30,271 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bd6592c37bb2:0-2 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732486585339.8ca644d2cada6dc44b363b7383dcd4fe. 2024-11-24T22:16:30,272 INFO [RS_FLUSH_OPERATIONS-regionserver/bd6592c37bb2:0-2 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.HRegion(2902): Flushing 8ca644d2cada6dc44b363b7383dcd4fe 3/3 column families, dataSize=136.42 KB heapSize=380.69 KB 2024-11-24T22:16:30,272 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bd6592c37bb2:0-2 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 8ca644d2cada6dc44b363b7383dcd4fe, store=A 2024-11-24T22:16:30,272 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bd6592c37bb2:0-2 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-24T22:16:30,272 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bd6592c37bb2:0-2 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 8ca644d2cada6dc44b363b7383dcd4fe, store=B 2024-11-24T22:16:30,272 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bd6592c37bb2:0-2 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-24T22:16:30,272 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bd6592c37bb2:0-2 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 8ca644d2cada6dc44b363b7383dcd4fe, store=C 2024-11-24T22:16:30,272 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bd6592c37bb2:0-2 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-24T22:16:30,279 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bd6592c37bb2:0-2 {event_type=RS_FLUSH_REGIONS, pid=153}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411248adf33bdcf2d4bad9a3cca7110343f1c_8ca644d2cada6dc44b363b7383dcd4fe is 50, key is test_row_0/A:col10/1732486588531/Put/seqid=0 2024-11-24T22:16:30,283 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45929 is added to blk_1073742390_1566 (size=12154) 2024-11-24T22:16:30,283 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bd6592c37bb2:0-2 {event_type=RS_FLUSH_REGIONS, pid=153}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T22:16:30,288 INFO [RS_FLUSH_OPERATIONS-regionserver/bd6592c37bb2:0-2 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411248adf33bdcf2d4bad9a3cca7110343f1c_8ca644d2cada6dc44b363b7383dcd4fe to hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411248adf33bdcf2d4bad9a3cca7110343f1c_8ca644d2cada6dc44b363b7383dcd4fe 2024-11-24T22:16:30,289 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bd6592c37bb2:0-2 {event_type=RS_FLUSH_REGIONS, pid=153}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/8ca644d2cada6dc44b363b7383dcd4fe/.tmp/A/35a85100955447888670863594d84360, store: [table=TestAcidGuarantees family=A region=8ca644d2cada6dc44b363b7383dcd4fe] 2024-11-24T22:16:30,289 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bd6592c37bb2:0-2 {event_type=RS_FLUSH_REGIONS, pid=153}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/8ca644d2cada6dc44b363b7383dcd4fe/.tmp/A/35a85100955447888670863594d84360 is 175, key is test_row_0/A:col10/1732486588531/Put/seqid=0 2024-11-24T22:16:30,293 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45929 is added to blk_1073742391_1567 (size=30955) 2024-11-24T22:16:30,591 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37701 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=152 2024-11-24T22:16:30,684 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40087 {}] regionserver.HRegion(8855): Flush requested on 8ca644d2cada6dc44b363b7383dcd4fe 2024-11-24T22:16:30,684 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732486585339.8ca644d2cada6dc44b363b7383dcd4fe. as already flushing 2024-11-24T22:16:30,693 INFO [RS_FLUSH_OPERATIONS-regionserver/bd6592c37bb2:0-2 {event_type=RS_FLUSH_REGIONS, pid=153}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=49, memsize=47.0 K, hasBloomFilter=true, into tmp file hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/8ca644d2cada6dc44b363b7383dcd4fe/.tmp/A/35a85100955447888670863594d84360 2024-11-24T22:16:30,698 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40087 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8ca644d2cada6dc44b363b7383dcd4fe, server=bd6592c37bb2,40087,1732486423713 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-24T22:16:30,698 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40087 {}] ipc.CallRunner(138): callId: 17 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48986 deadline: 1732486650693, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8ca644d2cada6dc44b363b7383dcd4fe, server=bd6592c37bb2,40087,1732486423713 2024-11-24T22:16:30,699 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40087 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8ca644d2cada6dc44b363b7383dcd4fe, server=bd6592c37bb2,40087,1732486423713 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-24T22:16:30,699 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732486585339.8ca644d2cada6dc44b363b7383dcd4fe., hostname=bd6592c37bb2,40087,1732486423713, seqNum=5 , the old value is region=TestAcidGuarantees,,1732486585339.8ca644d2cada6dc44b363b7383dcd4fe., hostname=bd6592c37bb2,40087,1732486423713, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8ca644d2cada6dc44b363b7383dcd4fe, server=bd6592c37bb2,40087,1732486423713 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T22:16:30,699 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40087 {}] ipc.CallRunner(138): callId: 18 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49026 deadline: 1732486650696, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8ca644d2cada6dc44b363b7383dcd4fe, server=bd6592c37bb2,40087,1732486423713 2024-11-24T22:16:30,699 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732486585339.8ca644d2cada6dc44b363b7383dcd4fe., hostname=bd6592c37bb2,40087,1732486423713, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8ca644d2cada6dc44b363b7383dcd4fe, server=bd6592c37bb2,40087,1732486423713 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T22:16:30,699 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732486585339.8ca644d2cada6dc44b363b7383dcd4fe., hostname=bd6592c37bb2,40087,1732486423713, seqNum=5 because the exception is null or not the one we care about 2024-11-24T22:16:30,700 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bd6592c37bb2:0-2 {event_type=RS_FLUSH_REGIONS, pid=153}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/8ca644d2cada6dc44b363b7383dcd4fe/.tmp/B/c807ab7919a94e47a5f40b2f71d3a197 is 50, key is test_row_0/B:col10/1732486588531/Put/seqid=0 2024-11-24T22:16:30,701 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732486585339.8ca644d2cada6dc44b363b7383dcd4fe., hostname=bd6592c37bb2,40087,1732486423713, seqNum=5 , the old value is region=TestAcidGuarantees,,1732486585339.8ca644d2cada6dc44b363b7383dcd4fe., hostname=bd6592c37bb2,40087,1732486423713, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8ca644d2cada6dc44b363b7383dcd4fe, server=bd6592c37bb2,40087,1732486423713 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T22:16:30,701 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732486585339.8ca644d2cada6dc44b363b7383dcd4fe., hostname=bd6592c37bb2,40087,1732486423713, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8ca644d2cada6dc44b363b7383dcd4fe, server=bd6592c37bb2,40087,1732486423713 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T22:16:30,701 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732486585339.8ca644d2cada6dc44b363b7383dcd4fe., hostname=bd6592c37bb2,40087,1732486423713, seqNum=5 because the exception is null or not the one we care about 2024-11-24T22:16:30,703 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40087 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8ca644d2cada6dc44b363b7383dcd4fe, server=bd6592c37bb2,40087,1732486423713 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-24T22:16:30,704 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40087 {}] ipc.CallRunner(138): callId: 16 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49002 deadline: 1732486650698, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8ca644d2cada6dc44b363b7383dcd4fe, server=bd6592c37bb2,40087,1732486423713 2024-11-24T22:16:30,704 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40087 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8ca644d2cada6dc44b363b7383dcd4fe, server=bd6592c37bb2,40087,1732486423713 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-24T22:16:30,704 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40087 {}] ipc.CallRunner(138): callId: 13 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49060 deadline: 1732486650699, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8ca644d2cada6dc44b363b7383dcd4fe, server=bd6592c37bb2,40087,1732486423713 2024-11-24T22:16:30,704 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732486585339.8ca644d2cada6dc44b363b7383dcd4fe., hostname=bd6592c37bb2,40087,1732486423713, seqNum=5 , the old value is region=TestAcidGuarantees,,1732486585339.8ca644d2cada6dc44b363b7383dcd4fe., hostname=bd6592c37bb2,40087,1732486423713, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8ca644d2cada6dc44b363b7383dcd4fe, server=bd6592c37bb2,40087,1732486423713 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T22:16:30,704 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732486585339.8ca644d2cada6dc44b363b7383dcd4fe., hostname=bd6592c37bb2,40087,1732486423713, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8ca644d2cada6dc44b363b7383dcd4fe, server=bd6592c37bb2,40087,1732486423713 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T22:16:30,704 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40087 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8ca644d2cada6dc44b363b7383dcd4fe, server=bd6592c37bb2,40087,1732486423713 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-24T22:16:30,704 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732486585339.8ca644d2cada6dc44b363b7383dcd4fe., hostname=bd6592c37bb2,40087,1732486423713, seqNum=5 because the exception is null or not the one we care about 2024-11-24T22:16:30,704 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40087 {}] ipc.CallRunner(138): callId: 13 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49050 deadline: 1732486650700, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8ca644d2cada6dc44b363b7383dcd4fe, server=bd6592c37bb2,40087,1732486423713 2024-11-24T22:16:30,704 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732486585339.8ca644d2cada6dc44b363b7383dcd4fe., hostname=bd6592c37bb2,40087,1732486423713, seqNum=5 , the old value is region=TestAcidGuarantees,,1732486585339.8ca644d2cada6dc44b363b7383dcd4fe., hostname=bd6592c37bb2,40087,1732486423713, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8ca644d2cada6dc44b363b7383dcd4fe, server=bd6592c37bb2,40087,1732486423713 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T22:16:30,704 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732486585339.8ca644d2cada6dc44b363b7383dcd4fe., hostname=bd6592c37bb2,40087,1732486423713, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8ca644d2cada6dc44b363b7383dcd4fe, server=bd6592c37bb2,40087,1732486423713 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T22:16:30,704 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732486585339.8ca644d2cada6dc44b363b7383dcd4fe., hostname=bd6592c37bb2,40087,1732486423713, seqNum=5 because the exception is null or not the one we care about 2024-11-24T22:16:30,704 WARN [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRpcRetryingCaller(177): Call to bd6592c37bb2,40087,1732486423713 for 'test_row_0' in 8ca644d2cada6dc44b363b7383dcd4fe of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2171 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8ca644d2cada6dc44b363b7383dcd4fe, server=bd6592c37bb2,40087,1732486423713 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor44.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8ca644d2cada6dc44b363b7383dcd4fe, server=bd6592c37bb2,40087,1732486423713 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-24T22:16:30,705 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732486585339.8ca644d2cada6dc44b363b7383dcd4fe., hostname=bd6592c37bb2,40087,1732486423713, seqNum=5 , the old value is region=TestAcidGuarantees,,1732486585339.8ca644d2cada6dc44b363b7383dcd4fe., hostname=bd6592c37bb2,40087,1732486423713, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8ca644d2cada6dc44b363b7383dcd4fe, server=bd6592c37bb2,40087,1732486423713 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T22:16:30,705 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732486585339.8ca644d2cada6dc44b363b7383dcd4fe., hostname=bd6592c37bb2,40087,1732486423713, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8ca644d2cada6dc44b363b7383dcd4fe, server=bd6592c37bb2,40087,1732486423713 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T22:16:30,705 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732486585339.8ca644d2cada6dc44b363b7383dcd4fe., hostname=bd6592c37bb2,40087,1732486423713, seqNum=5 because the exception is null or not the one we care about 2024-11-24T22:16:30,706 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45929 is added to blk_1073742392_1568 (size=12001) 2024-11-24T22:16:30,804 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40087 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8ca644d2cada6dc44b363b7383dcd4fe, server=bd6592c37bb2,40087,1732486423713 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-24T22:16:30,805 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40087 {}] ipc.CallRunner(138): callId: 18 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48986 deadline: 1732486650801, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8ca644d2cada6dc44b363b7383dcd4fe, server=bd6592c37bb2,40087,1732486423713 2024-11-24T22:16:30,805 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732486585339.8ca644d2cada6dc44b363b7383dcd4fe., hostname=bd6592c37bb2,40087,1732486423713, seqNum=5 , the old value is region=TestAcidGuarantees,,1732486585339.8ca644d2cada6dc44b363b7383dcd4fe., hostname=bd6592c37bb2,40087,1732486423713, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8ca644d2cada6dc44b363b7383dcd4fe, server=bd6592c37bb2,40087,1732486423713 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T22:16:30,805 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732486585339.8ca644d2cada6dc44b363b7383dcd4fe., hostname=bd6592c37bb2,40087,1732486423713, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8ca644d2cada6dc44b363b7383dcd4fe, server=bd6592c37bb2,40087,1732486423713 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T22:16:30,805 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732486585339.8ca644d2cada6dc44b363b7383dcd4fe., hostname=bd6592c37bb2,40087,1732486423713, seqNum=5 because the exception is null or not the one we care about 2024-11-24T22:16:30,810 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40087 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8ca644d2cada6dc44b363b7383dcd4fe, server=bd6592c37bb2,40087,1732486423713 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-24T22:16:30,811 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40087 {}] ipc.CallRunner(138): callId: 19 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49026 deadline: 1732486650810, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8ca644d2cada6dc44b363b7383dcd4fe, server=bd6592c37bb2,40087,1732486423713 2024-11-24T22:16:30,811 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40087 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8ca644d2cada6dc44b363b7383dcd4fe, server=bd6592c37bb2,40087,1732486423713 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-24T22:16:30,811 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732486585339.8ca644d2cada6dc44b363b7383dcd4fe., hostname=bd6592c37bb2,40087,1732486423713, seqNum=5 , the old value is region=TestAcidGuarantees,,1732486585339.8ca644d2cada6dc44b363b7383dcd4fe., hostname=bd6592c37bb2,40087,1732486423713, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8ca644d2cada6dc44b363b7383dcd4fe, server=bd6592c37bb2,40087,1732486423713 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T22:16:30,811 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732486585339.8ca644d2cada6dc44b363b7383dcd4fe., hostname=bd6592c37bb2,40087,1732486423713, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8ca644d2cada6dc44b363b7383dcd4fe, server=bd6592c37bb2,40087,1732486423713 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T22:16:30,811 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40087 {}] ipc.CallRunner(138): callId: 14 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49060 deadline: 1732486650811, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8ca644d2cada6dc44b363b7383dcd4fe, server=bd6592c37bb2,40087,1732486423713 2024-11-24T22:16:30,811 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732486585339.8ca644d2cada6dc44b363b7383dcd4fe., hostname=bd6592c37bb2,40087,1732486423713, seqNum=5 because the exception is null or not the one we care about 2024-11-24T22:16:30,811 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40087 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8ca644d2cada6dc44b363b7383dcd4fe, server=bd6592c37bb2,40087,1732486423713 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-24T22:16:30,811 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732486585339.8ca644d2cada6dc44b363b7383dcd4fe., hostname=bd6592c37bb2,40087,1732486423713, seqNum=5 , the old value is region=TestAcidGuarantees,,1732486585339.8ca644d2cada6dc44b363b7383dcd4fe., hostname=bd6592c37bb2,40087,1732486423713, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8ca644d2cada6dc44b363b7383dcd4fe, server=bd6592c37bb2,40087,1732486423713 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T22:16:30,811 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732486585339.8ca644d2cada6dc44b363b7383dcd4fe., hostname=bd6592c37bb2,40087,1732486423713, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8ca644d2cada6dc44b363b7383dcd4fe, server=bd6592c37bb2,40087,1732486423713 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T22:16:30,811 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40087 {}] ipc.CallRunner(138): callId: 17 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49002 deadline: 1732486650811, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8ca644d2cada6dc44b363b7383dcd4fe, server=bd6592c37bb2,40087,1732486423713 2024-11-24T22:16:30,811 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732486585339.8ca644d2cada6dc44b363b7383dcd4fe., hostname=bd6592c37bb2,40087,1732486423713, seqNum=5 because the exception is null or not the one we care about 2024-11-24T22:16:30,812 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732486585339.8ca644d2cada6dc44b363b7383dcd4fe., hostname=bd6592c37bb2,40087,1732486423713, seqNum=5 , the old value is region=TestAcidGuarantees,,1732486585339.8ca644d2cada6dc44b363b7383dcd4fe., hostname=bd6592c37bb2,40087,1732486423713, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8ca644d2cada6dc44b363b7383dcd4fe, server=bd6592c37bb2,40087,1732486423713 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T22:16:30,812 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732486585339.8ca644d2cada6dc44b363b7383dcd4fe., hostname=bd6592c37bb2,40087,1732486423713, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8ca644d2cada6dc44b363b7383dcd4fe, server=bd6592c37bb2,40087,1732486423713 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T22:16:30,812 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732486585339.8ca644d2cada6dc44b363b7383dcd4fe., hostname=bd6592c37bb2,40087,1732486423713, seqNum=5 because the exception is null or not the one we care about 2024-11-24T22:16:31,015 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40087 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8ca644d2cada6dc44b363b7383dcd4fe, server=bd6592c37bb2,40087,1732486423713 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-24T22:16:31,015 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40087 {}] ipc.CallRunner(138): callId: 19 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48986 deadline: 1732486651011, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8ca644d2cada6dc44b363b7383dcd4fe, server=bd6592c37bb2,40087,1732486423713 2024-11-24T22:16:31,015 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732486585339.8ca644d2cada6dc44b363b7383dcd4fe., hostname=bd6592c37bb2,40087,1732486423713, seqNum=5 , the old value is region=TestAcidGuarantees,,1732486585339.8ca644d2cada6dc44b363b7383dcd4fe., hostname=bd6592c37bb2,40087,1732486423713, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8ca644d2cada6dc44b363b7383dcd4fe, server=bd6592c37bb2,40087,1732486423713 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T22:16:31,015 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732486585339.8ca644d2cada6dc44b363b7383dcd4fe., hostname=bd6592c37bb2,40087,1732486423713, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8ca644d2cada6dc44b363b7383dcd4fe, server=bd6592c37bb2,40087,1732486423713 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T22:16:31,016 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732486585339.8ca644d2cada6dc44b363b7383dcd4fe., hostname=bd6592c37bb2,40087,1732486423713, seqNum=5 because the exception is null or not the one we care about 2024-11-24T22:16:31,024 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40087 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8ca644d2cada6dc44b363b7383dcd4fe, server=bd6592c37bb2,40087,1732486423713 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-24T22:16:31,024 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40087 {}] ipc.CallRunner(138): callId: 20 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49026 deadline: 1732486651020, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8ca644d2cada6dc44b363b7383dcd4fe, server=bd6592c37bb2,40087,1732486423713 2024-11-24T22:16:31,024 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40087 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8ca644d2cada6dc44b363b7383dcd4fe, server=bd6592c37bb2,40087,1732486423713 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-24T22:16:31,024 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40087 {}] ipc.CallRunner(138): callId: 15 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49060 deadline: 1732486651020, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8ca644d2cada6dc44b363b7383dcd4fe, server=bd6592c37bb2,40087,1732486423713 2024-11-24T22:16:31,024 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732486585339.8ca644d2cada6dc44b363b7383dcd4fe., hostname=bd6592c37bb2,40087,1732486423713, seqNum=5 , the old value is region=TestAcidGuarantees,,1732486585339.8ca644d2cada6dc44b363b7383dcd4fe., hostname=bd6592c37bb2,40087,1732486423713, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8ca644d2cada6dc44b363b7383dcd4fe, server=bd6592c37bb2,40087,1732486423713 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T22:16:31,024 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732486585339.8ca644d2cada6dc44b363b7383dcd4fe., hostname=bd6592c37bb2,40087,1732486423713, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8ca644d2cada6dc44b363b7383dcd4fe, server=bd6592c37bb2,40087,1732486423713 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T22:16:31,024 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732486585339.8ca644d2cada6dc44b363b7383dcd4fe., hostname=bd6592c37bb2,40087,1732486423713, seqNum=5 because the exception is null or not the one we care about 2024-11-24T22:16:31,024 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732486585339.8ca644d2cada6dc44b363b7383dcd4fe., hostname=bd6592c37bb2,40087,1732486423713, seqNum=5 , the old value is region=TestAcidGuarantees,,1732486585339.8ca644d2cada6dc44b363b7383dcd4fe., hostname=bd6592c37bb2,40087,1732486423713, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8ca644d2cada6dc44b363b7383dcd4fe, server=bd6592c37bb2,40087,1732486423713 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T22:16:31,024 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40087 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8ca644d2cada6dc44b363b7383dcd4fe, server=bd6592c37bb2,40087,1732486423713 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-24T22:16:31,024 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732486585339.8ca644d2cada6dc44b363b7383dcd4fe., hostname=bd6592c37bb2,40087,1732486423713, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8ca644d2cada6dc44b363b7383dcd4fe, server=bd6592c37bb2,40087,1732486423713 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T22:16:31,024 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732486585339.8ca644d2cada6dc44b363b7383dcd4fe., hostname=bd6592c37bb2,40087,1732486423713, seqNum=5 because the exception is null or not the one we care about 2024-11-24T22:16:31,024 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40087 {}] ipc.CallRunner(138): callId: 18 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49002 deadline: 1732486651021, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8ca644d2cada6dc44b363b7383dcd4fe, server=bd6592c37bb2,40087,1732486423713 2024-11-24T22:16:31,025 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732486585339.8ca644d2cada6dc44b363b7383dcd4fe., hostname=bd6592c37bb2,40087,1732486423713, seqNum=5 , the old value is region=TestAcidGuarantees,,1732486585339.8ca644d2cada6dc44b363b7383dcd4fe., hostname=bd6592c37bb2,40087,1732486423713, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8ca644d2cada6dc44b363b7383dcd4fe, server=bd6592c37bb2,40087,1732486423713 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T22:16:31,025 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732486585339.8ca644d2cada6dc44b363b7383dcd4fe., hostname=bd6592c37bb2,40087,1732486423713, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8ca644d2cada6dc44b363b7383dcd4fe, server=bd6592c37bb2,40087,1732486423713 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T22:16:31,025 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732486585339.8ca644d2cada6dc44b363b7383dcd4fe., hostname=bd6592c37bb2,40087,1732486423713, seqNum=5 because the exception is null or not the one we care about 2024-11-24T22:16:31,107 INFO [RS_FLUSH_OPERATIONS-regionserver/bd6592c37bb2:0-2 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=46.96 KB at sequenceid=49 (bloomFilter=true), to=hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/8ca644d2cada6dc44b363b7383dcd4fe/.tmp/B/c807ab7919a94e47a5f40b2f71d3a197 2024-11-24T22:16:31,114 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bd6592c37bb2:0-2 {event_type=RS_FLUSH_REGIONS, pid=153}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/8ca644d2cada6dc44b363b7383dcd4fe/.tmp/C/63f9798f78214b7ab82aae257603936a is 50, key is test_row_0/C:col10/1732486588531/Put/seqid=0 2024-11-24T22:16:31,117 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45929 is added to blk_1073742393_1569 (size=12001) 2024-11-24T22:16:31,321 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40087 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8ca644d2cada6dc44b363b7383dcd4fe, server=bd6592c37bb2,40087,1732486423713 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-24T22:16:31,321 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40087 {}] ipc.CallRunner(138): callId: 20 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48986 deadline: 1732486651321, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8ca644d2cada6dc44b363b7383dcd4fe, server=bd6592c37bb2,40087,1732486423713 2024-11-24T22:16:31,322 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732486585339.8ca644d2cada6dc44b363b7383dcd4fe., hostname=bd6592c37bb2,40087,1732486423713, seqNum=5 , the old value is region=TestAcidGuarantees,,1732486585339.8ca644d2cada6dc44b363b7383dcd4fe., hostname=bd6592c37bb2,40087,1732486423713, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8ca644d2cada6dc44b363b7383dcd4fe, server=bd6592c37bb2,40087,1732486423713 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T22:16:31,322 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732486585339.8ca644d2cada6dc44b363b7383dcd4fe., hostname=bd6592c37bb2,40087,1732486423713, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8ca644d2cada6dc44b363b7383dcd4fe, server=bd6592c37bb2,40087,1732486423713 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T22:16:31,322 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732486585339.8ca644d2cada6dc44b363b7383dcd4fe., hostname=bd6592c37bb2,40087,1732486423713, seqNum=5 because the exception is null or not the one we care about 2024-11-24T22:16:31,335 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40087 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8ca644d2cada6dc44b363b7383dcd4fe, server=bd6592c37bb2,40087,1732486423713 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-24T22:16:31,335 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40087 {}] ipc.CallRunner(138): callId: 21 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49026 deadline: 1732486651331, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8ca644d2cada6dc44b363b7383dcd4fe, server=bd6592c37bb2,40087,1732486423713 2024-11-24T22:16:31,335 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40087 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8ca644d2cada6dc44b363b7383dcd4fe, server=bd6592c37bb2,40087,1732486423713 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-24T22:16:31,335 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732486585339.8ca644d2cada6dc44b363b7383dcd4fe., hostname=bd6592c37bb2,40087,1732486423713, seqNum=5 , the old value is region=TestAcidGuarantees,,1732486585339.8ca644d2cada6dc44b363b7383dcd4fe., hostname=bd6592c37bb2,40087,1732486423713, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8ca644d2cada6dc44b363b7383dcd4fe, server=bd6592c37bb2,40087,1732486423713 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T22:16:31,335 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732486585339.8ca644d2cada6dc44b363b7383dcd4fe., hostname=bd6592c37bb2,40087,1732486423713, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8ca644d2cada6dc44b363b7383dcd4fe, server=bd6592c37bb2,40087,1732486423713 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T22:16:31,335 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40087 {}] ipc.CallRunner(138): callId: 16 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49060 deadline: 1732486651331, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8ca644d2cada6dc44b363b7383dcd4fe, server=bd6592c37bb2,40087,1732486423713 2024-11-24T22:16:31,335 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732486585339.8ca644d2cada6dc44b363b7383dcd4fe., hostname=bd6592c37bb2,40087,1732486423713, seqNum=5 because the exception is null or not the one we care about 2024-11-24T22:16:31,335 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40087 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8ca644d2cada6dc44b363b7383dcd4fe, server=bd6592c37bb2,40087,1732486423713 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-24T22:16:31,335 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732486585339.8ca644d2cada6dc44b363b7383dcd4fe., hostname=bd6592c37bb2,40087,1732486423713, seqNum=5 , the old value is region=TestAcidGuarantees,,1732486585339.8ca644d2cada6dc44b363b7383dcd4fe., hostname=bd6592c37bb2,40087,1732486423713, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8ca644d2cada6dc44b363b7383dcd4fe, server=bd6592c37bb2,40087,1732486423713 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T22:16:31,335 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40087 {}] ipc.CallRunner(138): callId: 19 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49002 deadline: 1732486651331, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8ca644d2cada6dc44b363b7383dcd4fe, server=bd6592c37bb2,40087,1732486423713 2024-11-24T22:16:31,335 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732486585339.8ca644d2cada6dc44b363b7383dcd4fe., hostname=bd6592c37bb2,40087,1732486423713, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8ca644d2cada6dc44b363b7383dcd4fe, server=bd6592c37bb2,40087,1732486423713 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T22:16:31,335 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732486585339.8ca644d2cada6dc44b363b7383dcd4fe., hostname=bd6592c37bb2,40087,1732486423713, seqNum=5 because the exception is null or not the one we care about 2024-11-24T22:16:31,336 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732486585339.8ca644d2cada6dc44b363b7383dcd4fe., hostname=bd6592c37bb2,40087,1732486423713, seqNum=5 , the old value is region=TestAcidGuarantees,,1732486585339.8ca644d2cada6dc44b363b7383dcd4fe., hostname=bd6592c37bb2,40087,1732486423713, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8ca644d2cada6dc44b363b7383dcd4fe, server=bd6592c37bb2,40087,1732486423713 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T22:16:31,336 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732486585339.8ca644d2cada6dc44b363b7383dcd4fe., hostname=bd6592c37bb2,40087,1732486423713, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8ca644d2cada6dc44b363b7383dcd4fe, server=bd6592c37bb2,40087,1732486423713 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T22:16:31,336 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732486585339.8ca644d2cada6dc44b363b7383dcd4fe., hostname=bd6592c37bb2,40087,1732486423713, seqNum=5 because the exception is null or not the one we care about 2024-11-24T22:16:31,517 INFO [RS_FLUSH_OPERATIONS-regionserver/bd6592c37bb2:0-2 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=42.49 KB at sequenceid=49 (bloomFilter=true), to=hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/8ca644d2cada6dc44b363b7383dcd4fe/.tmp/C/63f9798f78214b7ab82aae257603936a 2024-11-24T22:16:31,521 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bd6592c37bb2:0-2 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/8ca644d2cada6dc44b363b7383dcd4fe/.tmp/A/35a85100955447888670863594d84360 as hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/8ca644d2cada6dc44b363b7383dcd4fe/A/35a85100955447888670863594d84360 2024-11-24T22:16:31,524 INFO [RS_FLUSH_OPERATIONS-regionserver/bd6592c37bb2:0-2 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/8ca644d2cada6dc44b363b7383dcd4fe/A/35a85100955447888670863594d84360, entries=150, sequenceid=49, filesize=30.2 K 2024-11-24T22:16:31,525 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bd6592c37bb2:0-2 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/8ca644d2cada6dc44b363b7383dcd4fe/.tmp/B/c807ab7919a94e47a5f40b2f71d3a197 as hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/8ca644d2cada6dc44b363b7383dcd4fe/B/c807ab7919a94e47a5f40b2f71d3a197 2024-11-24T22:16:31,528 INFO [RS_FLUSH_OPERATIONS-regionserver/bd6592c37bb2:0-2 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/8ca644d2cada6dc44b363b7383dcd4fe/B/c807ab7919a94e47a5f40b2f71d3a197, entries=150, sequenceid=49, filesize=11.7 K 2024-11-24T22:16:31,528 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bd6592c37bb2:0-2 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/8ca644d2cada6dc44b363b7383dcd4fe/.tmp/C/63f9798f78214b7ab82aae257603936a as hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/8ca644d2cada6dc44b363b7383dcd4fe/C/63f9798f78214b7ab82aae257603936a 2024-11-24T22:16:31,530 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T22:16:31,532 INFO [RS_FLUSH_OPERATIONS-regionserver/bd6592c37bb2:0-2 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/8ca644d2cada6dc44b363b7383dcd4fe/C/63f9798f78214b7ab82aae257603936a, entries=150, sequenceid=49, filesize=11.7 K 2024-11-24T22:16:31,533 INFO [RS_FLUSH_OPERATIONS-regionserver/bd6592c37bb2:0-2 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.HRegion(3140): Finished flush of dataSize ~136.42 KB/139690, heapSize ~380.64 KB/389776, currentSize=53.67 KB/54960 for 8ca644d2cada6dc44b363b7383dcd4fe in 1260ms, sequenceid=49, compaction requested=false 2024-11-24T22:16:31,533 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bd6592c37bb2:0-2 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.HRegion(2603): Flush status journal for 8ca644d2cada6dc44b363b7383dcd4fe: 2024-11-24T22:16:31,533 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bd6592c37bb2:0-2 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732486585339.8ca644d2cada6dc44b363b7383dcd4fe. 2024-11-24T22:16:31,533 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bd6592c37bb2:0-2 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=153 2024-11-24T22:16:31,533 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37701 {}] master.HMaster(4169): Remote procedure done, pid=153 2024-11-24T22:16:31,535 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=153, resume processing ppid=152 2024-11-24T22:16:31,535 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=153, ppid=152, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 3.0960 sec 2024-11-24T22:16:31,537 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=152, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=152, table=TestAcidGuarantees in 3.1010 sec 2024-11-24T22:16:31,542 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T22:16:31,543 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T22:16:31,543 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T22:16:31,544 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T22:16:31,544 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T22:16:31,545 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T22:16:31,545 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T22:16:31,547 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T22:16:31,547 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T22:16:31,547 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T22:16:31,548 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T22:16:31,548 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T22:16:31,549 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T22:16:31,554 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T22:16:31,554 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T22:16:31,555 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T22:16:31,555 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T22:16:31,556 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T22:16:31,556 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T22:16:31,557 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T22:16:31,557 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T22:16:31,558 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T22:16:31,558 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T22:16:31,559 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T22:16:31,559 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T22:16:31,560 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T22:16:31,560 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T22:16:31,561 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T22:16:31,561 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T22:16:31,562 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T22:16:31,563 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T22:16:31,563 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T22:16:31,564 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T22:16:31,565 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T22:16:31,565 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T22:16:31,566 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T22:16:31,566 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T22:16:31,566 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T22:16:31,567 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T22:16:31,567 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T22:16:31,568 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T22:16:31,569 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T22:16:31,569 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T22:16:31,570 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T22:16:31,570 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T22:16:31,571 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T22:16:31,571 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T22:16:31,572 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T22:16:31,572 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T22:16:31,576 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T22:16:31,577 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T22:16:31,577 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T22:16:31,578 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T22:16:31,578 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T22:16:31,579 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T22:16:31,579 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T22:16:31,580 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T22:16:31,580 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T22:16:31,581 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T22:16:31,581 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T22:16:31,582 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T22:16:31,582 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T22:16:31,583 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T22:16:31,583 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T22:16:31,584 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T22:16:31,584 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T22:16:31,585 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T22:16:31,585 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T22:16:31,586 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T22:16:31,587 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T22:16:31,587 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T22:16:31,588 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T22:16:31,588 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T22:16:31,589 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T22:16:31,589 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T22:16:31,590 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T22:16:31,590 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T22:16:31,591 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T22:16:31,591 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T22:16:31,592 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T22:16:31,592 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T22:16:31,593 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T22:16:31,593 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T22:16:31,594 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T22:16:31,594 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T22:16:31,595 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T22:16:31,596 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T22:16:31,597 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T22:16:31,597 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T22:16:31,598 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T22:16:31,598 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T22:16:31,599 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T22:16:31,599 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T22:16:31,600 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T22:16:31,600 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T22:16:31,601 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T22:16:31,601 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T22:16:31,602 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T22:16:31,602 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T22:16:31,606 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T22:16:31,606 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T22:16:31,607 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T22:16:31,607 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T22:16:31,608 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T22:16:31,608 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T22:16:31,609 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T22:16:31,609 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T22:16:31,610 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T22:16:31,610 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T22:16:31,610 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T22:16:31,611 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T22:16:31,611 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T22:16:31,612 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T22:16:31,612 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T22:16:31,613 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T22:16:31,613 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T22:16:31,614 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T22:16:31,614 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T22:16:31,615 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T22:16:31,615 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T22:16:31,616 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T22:16:31,616 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T22:16:31,617 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T22:16:31,617 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T22:16:31,618 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T22:16:31,618 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T22:16:31,619 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T22:16:31,619 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T22:16:31,621 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T22:16:31,621 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T22:16:31,622 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T22:16:31,622 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T22:16:31,623 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T22:16:31,623 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T22:16:31,624 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T22:16:31,624 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T22:16:31,625 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T22:16:31,625 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T22:16:31,626 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T22:16:31,626 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T22:16:31,627 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T22:16:31,627 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T22:16:31,628 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T22:16:31,628 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T22:16:31,629 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T22:16:31,630 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T22:16:31,630 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T22:16:31,631 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T22:16:31,631 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T22:16:31,636 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T22:16:31,636 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T22:16:31,637 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T22:16:31,637 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T22:16:31,637 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T22:16:31,638 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T22:16:31,638 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T22:16:31,639 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T22:16:31,639 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T22:16:31,640 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T22:16:31,640 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T22:16:31,641 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T22:16:31,641 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T22:16:31,641 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T22:16:31,642 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T22:16:31,642 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T22:16:31,643 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T22:16:31,643 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T22:16:31,644 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T22:16:31,644 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T22:16:31,645 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T22:16:31,645 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T22:16:31,645 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T22:16:31,646 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T22:16:31,646 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T22:16:31,647 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T22:16:31,647 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T22:16:31,648 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T22:16:31,648 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T22:16:31,649 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T22:16:31,649 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T22:16:31,650 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T22:16:31,650 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T22:16:31,650 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T22:16:31,651 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T22:16:31,651 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T22:16:31,652 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T22:16:31,653 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T22:16:31,653 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T22:16:31,653 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T22:16:31,654 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T22:16:31,654 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T22:16:31,655 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T22:16:31,655 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T22:16:31,656 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T22:16:31,656 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T22:16:31,657 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T22:16:31,657 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T22:16:31,658 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T22:16:31,658 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T22:16:31,661 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T22:16:31,662 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T22:16:31,662 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T22:16:31,663 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T22:16:31,663 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T22:16:31,664 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T22:16:31,664 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T22:16:31,665 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T22:16:31,666 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T22:16:31,666 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T22:16:31,667 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T22:16:31,667 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T22:16:31,667 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T22:16:31,668 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T22:16:31,668 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T22:16:31,669 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T22:16:31,669 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T22:16:31,669 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T22:16:31,670 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T22:16:31,670 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T22:16:31,671 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T22:16:31,671 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T22:16:31,671 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T22:16:31,672 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T22:16:31,673 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T22:16:31,673 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T22:16:31,674 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T22:16:31,674 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T22:16:31,675 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T22:16:31,675 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T22:16:31,676 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T22:16:31,676 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T22:16:31,677 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T22:16:31,677 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T22:16:31,678 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T22:16:31,678 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T22:16:31,678 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T22:16:31,679 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T22:16:31,679 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T22:16:31,680 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T22:16:31,680 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T22:16:31,681 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T22:16:31,681 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T22:16:31,682 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T22:16:31,682 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T22:16:31,683 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T22:16:31,683 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T22:16:31,684 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T22:16:31,685 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T22:16:31,685 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T22:16:31,688 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T22:16:31,689 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T22:16:31,690 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T22:16:31,690 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T22:16:31,691 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T22:16:31,691 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T22:16:31,692 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T22:16:31,692 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T22:16:31,692 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T22:16:31,693 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T22:16:31,693 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T22:16:31,694 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T22:16:31,694 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T22:16:31,695 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T22:16:31,695 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T22:16:31,696 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T22:16:31,696 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T22:16:31,696 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T22:16:31,697 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T22:16:31,697 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T22:16:31,698 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T22:16:31,699 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T22:16:31,699 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T22:16:31,699 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T22:16:31,700 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T22:16:31,701 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T22:16:31,701 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T22:16:31,701 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T22:16:31,702 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T22:16:31,702 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T22:16:31,703 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T22:16:31,703 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T22:16:31,704 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T22:16:31,704 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T22:16:31,705 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T22:16:31,705 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T22:16:31,706 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T22:16:31,706 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T22:16:31,707 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T22:16:31,707 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T22:16:31,708 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T22:16:31,708 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T22:16:31,709 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T22:16:31,710 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T22:16:31,710 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T22:16:31,711 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T22:16:31,711 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T22:16:31,712 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T22:16:31,712 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T22:16:31,713 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T22:16:31,716 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T22:16:31,716 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T22:16:31,717 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T22:16:31,717 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T22:16:31,718 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T22:16:31,718 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T22:16:31,719 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T22:16:31,719 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T22:16:31,719 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T22:16:31,720 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T22:16:31,720 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T22:16:31,721 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T22:16:31,721 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T22:16:31,722 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T22:16:31,722 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T22:16:31,723 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T22:16:31,723 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T22:16:31,724 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T22:16:31,724 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T22:16:31,725 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T22:16:31,725 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T22:16:31,726 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T22:16:31,726 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T22:16:31,726 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T22:16:31,727 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T22:16:31,727 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T22:16:31,728 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T22:16:31,728 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T22:16:31,728 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T22:16:31,729 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T22:16:31,729 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T22:16:31,729 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T22:16:31,730 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T22:16:31,730 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T22:16:31,731 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T22:16:31,731 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T22:16:31,731 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T22:16:31,732 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T22:16:31,732 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T22:16:31,733 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T22:16:31,733 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T22:16:31,733 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T22:16:31,734 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T22:16:31,734 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T22:16:31,735 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T22:16:31,735 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T22:16:31,736 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T22:16:31,736 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T22:16:31,737 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T22:16:31,737 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T22:16:31,740 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T22:16:31,741 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T22:16:31,741 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T22:16:31,742 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T22:16:31,742 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T22:16:31,743 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T22:16:31,743 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T22:16:31,743 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T22:16:31,744 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T22:16:31,744 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T22:16:31,744 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T22:16:31,745 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T22:16:31,745 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T22:16:31,745 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T22:16:31,746 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T22:16:31,746 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T22:16:31,746 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T22:16:31,747 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T22:16:31,747 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T22:16:31,748 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T22:16:31,748 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T22:16:31,748 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T22:16:31,749 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T22:16:31,749 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T22:16:31,749 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T22:16:31,750 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T22:16:31,751 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T22:16:31,751 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T22:16:31,751 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T22:16:31,752 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T22:16:31,753 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T22:16:31,753 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T22:16:31,754 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T22:16:31,754 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T22:16:31,755 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T22:16:31,756 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T22:16:31,756 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T22:16:31,757 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T22:16:31,758 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T22:16:31,758 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T22:16:31,760 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T22:16:31,760 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T22:16:31,762 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T22:16:31,762 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T22:16:31,763 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T22:16:31,764 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T22:16:31,765 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T22:16:31,765 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T22:16:31,767 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T22:16:31,767 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T22:16:31,772 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T22:16:31,772 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T22:16:31,773 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T22:16:31,773 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T22:16:31,774 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T22:16:31,774 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T22:16:31,775 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T22:16:31,775 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T22:16:31,776 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T22:16:31,777 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T22:16:31,777 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T22:16:31,778 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T22:16:31,778 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T22:16:31,780 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T22:16:31,781 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T22:16:31,782 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T22:16:31,783 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T22:16:31,783 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T22:16:31,784 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T22:16:31,784 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T22:16:31,785 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T22:16:31,785 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T22:16:31,786 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T22:16:31,787 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T22:16:31,788 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T22:16:31,789 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T22:16:31,789 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T22:16:31,790 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T22:16:31,794 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T22:16:31,795 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T22:16:31,796 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T22:16:31,796 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T22:16:31,797 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T22:16:31,797 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T22:16:31,798 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T22:16:31,798 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T22:16:31,799 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T22:16:31,801 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T22:16:31,802 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T22:16:31,804 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T22:16:31,805 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T22:16:31,806 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T22:16:31,807 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T22:16:31,809 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T22:16:31,809 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T22:16:31,810 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T22:16:31,811 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T22:16:31,811 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T22:16:31,812 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T22:16:31,812 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T22:16:31,817 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T22:16:31,817 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T22:16:31,818 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T22:16:31,818 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T22:16:31,819 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T22:16:31,820 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T22:16:31,820 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T22:16:31,820 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T22:16:31,821 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T22:16:31,821 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T22:16:31,822 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T22:16:31,822 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T22:16:31,823 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T22:16:31,823 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T22:16:31,824 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T22:16:31,824 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T22:16:31,825 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T22:16:31,825 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T22:16:31,826 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T22:16:31,826 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T22:16:31,827 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T22:16:31,827 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T22:16:31,828 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T22:16:31,828 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T22:16:31,829 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T22:16:31,829 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T22:16:31,830 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T22:16:31,830 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T22:16:31,830 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T22:16:31,831 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T22:16:31,831 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T22:16:31,832 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T22:16:31,832 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T22:16:31,833 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T22:16:31,833 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T22:16:31,834 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T22:16:31,834 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T22:16:31,835 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40087 {}] regionserver.HRegion(8855): Flush requested on 8ca644d2cada6dc44b363b7383dcd4fe 2024-11-24T22:16:31,835 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T22:16:31,835 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 8ca644d2cada6dc44b363b7383dcd4fe 3/3 column families, dataSize=60.38 KB heapSize=180.05 KB 2024-11-24T22:16:31,835 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 8ca644d2cada6dc44b363b7383dcd4fe, store=A 2024-11-24T22:16:31,835 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-24T22:16:31,835 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 8ca644d2cada6dc44b363b7383dcd4fe, store=B 2024-11-24T22:16:31,835 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T22:16:31,835 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-24T22:16:31,835 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 8ca644d2cada6dc44b363b7383dcd4fe, store=C 2024-11-24T22:16:31,835 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-24T22:16:31,836 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T22:16:31,836 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T22:16:31,837 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T22:16:31,837 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T22:16:31,838 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T22:16:31,839 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T22:16:31,839 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T22:16:31,840 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T22:16:31,841 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T22:16:31,841 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241124f9c897fa992444198f55db8a31246739_8ca644d2cada6dc44b363b7383dcd4fe is 50, key is test_row_0/A:col10/1732486591833/Put/seqid=0 2024-11-24T22:16:31,841 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T22:16:31,842 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T22:16:31,854 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45929 is added to blk_1073742394_1570 (size=12154) 2024-11-24T22:16:31,875 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40087 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8ca644d2cada6dc44b363b7383dcd4fe, server=bd6592c37bb2,40087,1732486423713 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-24T22:16:31,875 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40087 {}] ipc.CallRunner(138): callId: 26 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49026 deadline: 1732486651869, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8ca644d2cada6dc44b363b7383dcd4fe, server=bd6592c37bb2,40087,1732486423713 2024-11-24T22:16:31,875 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40087 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8ca644d2cada6dc44b363b7383dcd4fe, server=bd6592c37bb2,40087,1732486423713 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-24T22:16:31,876 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40087 {}] ipc.CallRunner(138): callId: 21 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49060 deadline: 1732486651871, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8ca644d2cada6dc44b363b7383dcd4fe, server=bd6592c37bb2,40087,1732486423713 2024-11-24T22:16:31,876 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732486585339.8ca644d2cada6dc44b363b7383dcd4fe., hostname=bd6592c37bb2,40087,1732486423713, seqNum=5 , the old value is region=TestAcidGuarantees,,1732486585339.8ca644d2cada6dc44b363b7383dcd4fe., hostname=bd6592c37bb2,40087,1732486423713, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8ca644d2cada6dc44b363b7383dcd4fe, server=bd6592c37bb2,40087,1732486423713 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T22:16:31,876 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732486585339.8ca644d2cada6dc44b363b7383dcd4fe., hostname=bd6592c37bb2,40087,1732486423713, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8ca644d2cada6dc44b363b7383dcd4fe, server=bd6592c37bb2,40087,1732486423713 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T22:16:31,876 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732486585339.8ca644d2cada6dc44b363b7383dcd4fe., hostname=bd6592c37bb2,40087,1732486423713, seqNum=5 because the exception is null or not the one we care about 2024-11-24T22:16:31,876 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732486585339.8ca644d2cada6dc44b363b7383dcd4fe., hostname=bd6592c37bb2,40087,1732486423713, seqNum=5 , the old value is region=TestAcidGuarantees,,1732486585339.8ca644d2cada6dc44b363b7383dcd4fe., hostname=bd6592c37bb2,40087,1732486423713, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8ca644d2cada6dc44b363b7383dcd4fe, server=bd6592c37bb2,40087,1732486423713 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T22:16:31,876 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732486585339.8ca644d2cada6dc44b363b7383dcd4fe., hostname=bd6592c37bb2,40087,1732486423713, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8ca644d2cada6dc44b363b7383dcd4fe, server=bd6592c37bb2,40087,1732486423713 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T22:16:31,876 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732486585339.8ca644d2cada6dc44b363b7383dcd4fe., hostname=bd6592c37bb2,40087,1732486423713, seqNum=5 because the exception is null or not the one we care about 2024-11-24T22:16:31,877 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/bd6592c37bb2:0-4 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1732486591867, max timestamp=1732486591874Num uniques 859; 2024-11-24T22:16:31,877 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/bd6592c37bb2:0-4 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-11-24T22:16:31,877 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/bd6592c37bb2:0-0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1732486591867, max timestamp=1732486591874Num uniques 859; 2024-11-24T22:16:31,877 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/bd6592c37bb2:0-0 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-11-24T22:16:31,880 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/bd6592c37bb2:0-4 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1732486591867, max timestamp=1732486591874Num uniques 859; 2024-11-24T22:16:31,880 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/bd6592c37bb2:0-4 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-11-24T22:16:31,921 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40087 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8ca644d2cada6dc44b363b7383dcd4fe, server=bd6592c37bb2,40087,1732486423713 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-24T22:16:31,922 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40087 {}] ipc.CallRunner(138): callId: 35 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48986 deadline: 1732486651916, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8ca644d2cada6dc44b363b7383dcd4fe, server=bd6592c37bb2,40087,1732486423713 2024-11-24T22:16:31,922 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732486585339.8ca644d2cada6dc44b363b7383dcd4fe., hostname=bd6592c37bb2,40087,1732486423713, seqNum=5 , the old value is region=TestAcidGuarantees,,1732486585339.8ca644d2cada6dc44b363b7383dcd4fe., hostname=bd6592c37bb2,40087,1732486423713, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8ca644d2cada6dc44b363b7383dcd4fe, server=bd6592c37bb2,40087,1732486423713 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T22:16:31,922 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732486585339.8ca644d2cada6dc44b363b7383dcd4fe., hostname=bd6592c37bb2,40087,1732486423713, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8ca644d2cada6dc44b363b7383dcd4fe, server=bd6592c37bb2,40087,1732486423713 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T22:16:31,922 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732486585339.8ca644d2cada6dc44b363b7383dcd4fe., hostname=bd6592c37bb2,40087,1732486423713, seqNum=5 because the exception is null or not the one we care about 2024-11-24T22:16:31,926 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40087 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8ca644d2cada6dc44b363b7383dcd4fe, server=bd6592c37bb2,40087,1732486423713 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-24T22:16:31,926 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40087 {}] ipc.CallRunner(138): callId: 31 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49002 deadline: 1732486651921, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8ca644d2cada6dc44b363b7383dcd4fe, server=bd6592c37bb2,40087,1732486423713 2024-11-24T22:16:31,926 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732486585339.8ca644d2cada6dc44b363b7383dcd4fe., hostname=bd6592c37bb2,40087,1732486423713, seqNum=5 , the old value is region=TestAcidGuarantees,,1732486585339.8ca644d2cada6dc44b363b7383dcd4fe., hostname=bd6592c37bb2,40087,1732486423713, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8ca644d2cada6dc44b363b7383dcd4fe, server=bd6592c37bb2,40087,1732486423713 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T22:16:31,926 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732486585339.8ca644d2cada6dc44b363b7383dcd4fe., hostname=bd6592c37bb2,40087,1732486423713, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8ca644d2cada6dc44b363b7383dcd4fe, server=bd6592c37bb2,40087,1732486423713 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T22:16:31,926 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732486585339.8ca644d2cada6dc44b363b7383dcd4fe., hostname=bd6592c37bb2,40087,1732486423713, seqNum=5 because the exception is null or not the one we care about 2024-11-24T22:16:31,984 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40087 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8ca644d2cada6dc44b363b7383dcd4fe, server=bd6592c37bb2,40087,1732486423713 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-24T22:16:31,984 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40087 {}] ipc.CallRunner(138): callId: 27 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49026 deadline: 1732486651981, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8ca644d2cada6dc44b363b7383dcd4fe, server=bd6592c37bb2,40087,1732486423713 2024-11-24T22:16:31,984 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732486585339.8ca644d2cada6dc44b363b7383dcd4fe., hostname=bd6592c37bb2,40087,1732486423713, seqNum=5 , the old value is region=TestAcidGuarantees,,1732486585339.8ca644d2cada6dc44b363b7383dcd4fe., hostname=bd6592c37bb2,40087,1732486423713, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8ca644d2cada6dc44b363b7383dcd4fe, server=bd6592c37bb2,40087,1732486423713 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T22:16:31,984 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40087 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8ca644d2cada6dc44b363b7383dcd4fe, server=bd6592c37bb2,40087,1732486423713 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-24T22:16:31,985 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732486585339.8ca644d2cada6dc44b363b7383dcd4fe., hostname=bd6592c37bb2,40087,1732486423713, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8ca644d2cada6dc44b363b7383dcd4fe, server=bd6592c37bb2,40087,1732486423713 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T22:16:31,985 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732486585339.8ca644d2cada6dc44b363b7383dcd4fe., hostname=bd6592c37bb2,40087,1732486423713, seqNum=5 because the exception is null or not the one we care about 2024-11-24T22:16:31,985 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40087 {}] ipc.CallRunner(138): callId: 22 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49060 deadline: 1732486651981, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8ca644d2cada6dc44b363b7383dcd4fe, server=bd6592c37bb2,40087,1732486423713 2024-11-24T22:16:31,985 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732486585339.8ca644d2cada6dc44b363b7383dcd4fe., hostname=bd6592c37bb2,40087,1732486423713, seqNum=5 , the old value is region=TestAcidGuarantees,,1732486585339.8ca644d2cada6dc44b363b7383dcd4fe., hostname=bd6592c37bb2,40087,1732486423713, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8ca644d2cada6dc44b363b7383dcd4fe, server=bd6592c37bb2,40087,1732486423713 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T22:16:31,985 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732486585339.8ca644d2cada6dc44b363b7383dcd4fe., hostname=bd6592c37bb2,40087,1732486423713, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8ca644d2cada6dc44b363b7383dcd4fe, server=bd6592c37bb2,40087,1732486423713 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T22:16:31,985 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732486585339.8ca644d2cada6dc44b363b7383dcd4fe., hostname=bd6592c37bb2,40087,1732486423713, seqNum=5 because the exception is null or not the one we care about 2024-11-24T22:16:32,034 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40087 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8ca644d2cada6dc44b363b7383dcd4fe, server=bd6592c37bb2,40087,1732486423713 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-24T22:16:32,034 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40087 {}] ipc.CallRunner(138): callId: 36 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48986 deadline: 1732486652031, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8ca644d2cada6dc44b363b7383dcd4fe, server=bd6592c37bb2,40087,1732486423713 2024-11-24T22:16:32,034 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732486585339.8ca644d2cada6dc44b363b7383dcd4fe., hostname=bd6592c37bb2,40087,1732486423713, seqNum=5 , the old value is region=TestAcidGuarantees,,1732486585339.8ca644d2cada6dc44b363b7383dcd4fe., hostname=bd6592c37bb2,40087,1732486423713, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8ca644d2cada6dc44b363b7383dcd4fe, server=bd6592c37bb2,40087,1732486423713 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T22:16:32,034 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732486585339.8ca644d2cada6dc44b363b7383dcd4fe., hostname=bd6592c37bb2,40087,1732486423713, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8ca644d2cada6dc44b363b7383dcd4fe, server=bd6592c37bb2,40087,1732486423713 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T22:16:32,034 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732486585339.8ca644d2cada6dc44b363b7383dcd4fe., hostname=bd6592c37bb2,40087,1732486423713, seqNum=5 because the exception is null or not the one we care about 2024-11-24T22:16:32,035 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40087 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8ca644d2cada6dc44b363b7383dcd4fe, server=bd6592c37bb2,40087,1732486423713 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-24T22:16:32,035 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40087 {}] ipc.CallRunner(138): callId: 32 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49002 deadline: 1732486652031, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8ca644d2cada6dc44b363b7383dcd4fe, server=bd6592c37bb2,40087,1732486423713 2024-11-24T22:16:32,035 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732486585339.8ca644d2cada6dc44b363b7383dcd4fe., hostname=bd6592c37bb2,40087,1732486423713, seqNum=5 , the old value is region=TestAcidGuarantees,,1732486585339.8ca644d2cada6dc44b363b7383dcd4fe., hostname=bd6592c37bb2,40087,1732486423713, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8ca644d2cada6dc44b363b7383dcd4fe, server=bd6592c37bb2,40087,1732486423713 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T22:16:32,035 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732486585339.8ca644d2cada6dc44b363b7383dcd4fe., hostname=bd6592c37bb2,40087,1732486423713, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8ca644d2cada6dc44b363b7383dcd4fe, server=bd6592c37bb2,40087,1732486423713 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T22:16:32,035 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732486585339.8ca644d2cada6dc44b363b7383dcd4fe., hostname=bd6592c37bb2,40087,1732486423713, seqNum=5 because the exception is null or not the one we care about 2024-11-24T22:16:32,195 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40087 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8ca644d2cada6dc44b363b7383dcd4fe, server=bd6592c37bb2,40087,1732486423713 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-24T22:16:32,195 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40087 {}] ipc.CallRunner(138): callId: 28 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49026 deadline: 1732486652190, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8ca644d2cada6dc44b363b7383dcd4fe, server=bd6592c37bb2,40087,1732486423713 2024-11-24T22:16:32,195 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40087 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8ca644d2cada6dc44b363b7383dcd4fe, server=bd6592c37bb2,40087,1732486423713 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-24T22:16:32,195 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732486585339.8ca644d2cada6dc44b363b7383dcd4fe., hostname=bd6592c37bb2,40087,1732486423713, seqNum=5 , the old value is region=TestAcidGuarantees,,1732486585339.8ca644d2cada6dc44b363b7383dcd4fe., hostname=bd6592c37bb2,40087,1732486423713, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8ca644d2cada6dc44b363b7383dcd4fe, server=bd6592c37bb2,40087,1732486423713 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T22:16:32,195 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732486585339.8ca644d2cada6dc44b363b7383dcd4fe., hostname=bd6592c37bb2,40087,1732486423713, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8ca644d2cada6dc44b363b7383dcd4fe, server=bd6592c37bb2,40087,1732486423713 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T22:16:32,195 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40087 {}] ipc.CallRunner(138): callId: 23 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49060 deadline: 1732486652191, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8ca644d2cada6dc44b363b7383dcd4fe, server=bd6592c37bb2,40087,1732486423713 2024-11-24T22:16:32,195 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732486585339.8ca644d2cada6dc44b363b7383dcd4fe., hostname=bd6592c37bb2,40087,1732486423713, seqNum=5 because the exception is null or not the one we care about 2024-11-24T22:16:32,196 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732486585339.8ca644d2cada6dc44b363b7383dcd4fe., hostname=bd6592c37bb2,40087,1732486423713, seqNum=5 , the old value is region=TestAcidGuarantees,,1732486585339.8ca644d2cada6dc44b363b7383dcd4fe., hostname=bd6592c37bb2,40087,1732486423713, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8ca644d2cada6dc44b363b7383dcd4fe, server=bd6592c37bb2,40087,1732486423713 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T22:16:32,196 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732486585339.8ca644d2cada6dc44b363b7383dcd4fe., hostname=bd6592c37bb2,40087,1732486423713, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8ca644d2cada6dc44b363b7383dcd4fe, server=bd6592c37bb2,40087,1732486423713 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T22:16:32,196 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732486585339.8ca644d2cada6dc44b363b7383dcd4fe., hostname=bd6592c37bb2,40087,1732486423713, seqNum=5 because the exception is null or not the one we care about 2024-11-24T22:16:32,245 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40087 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8ca644d2cada6dc44b363b7383dcd4fe, server=bd6592c37bb2,40087,1732486423713 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-24T22:16:32,245 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40087 {}] ipc.CallRunner(138): callId: 37 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48986 deadline: 1732486652241, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8ca644d2cada6dc44b363b7383dcd4fe, server=bd6592c37bb2,40087,1732486423713 2024-11-24T22:16:32,245 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40087 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8ca644d2cada6dc44b363b7383dcd4fe, server=bd6592c37bb2,40087,1732486423713 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-24T22:16:32,245 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732486585339.8ca644d2cada6dc44b363b7383dcd4fe., hostname=bd6592c37bb2,40087,1732486423713, seqNum=5 , the old value is region=TestAcidGuarantees,,1732486585339.8ca644d2cada6dc44b363b7383dcd4fe., hostname=bd6592c37bb2,40087,1732486423713, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8ca644d2cada6dc44b363b7383dcd4fe, server=bd6592c37bb2,40087,1732486423713 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T22:16:32,245 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732486585339.8ca644d2cada6dc44b363b7383dcd4fe., hostname=bd6592c37bb2,40087,1732486423713, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8ca644d2cada6dc44b363b7383dcd4fe, server=bd6592c37bb2,40087,1732486423713 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T22:16:32,245 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40087 {}] ipc.CallRunner(138): callId: 33 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49002 deadline: 1732486652241, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8ca644d2cada6dc44b363b7383dcd4fe, server=bd6592c37bb2,40087,1732486423713 2024-11-24T22:16:32,245 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732486585339.8ca644d2cada6dc44b363b7383dcd4fe., hostname=bd6592c37bb2,40087,1732486423713, seqNum=5 because the exception is null or not the one we care about 2024-11-24T22:16:32,245 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732486585339.8ca644d2cada6dc44b363b7383dcd4fe., hostname=bd6592c37bb2,40087,1732486423713, seqNum=5 , the old value is region=TestAcidGuarantees,,1732486585339.8ca644d2cada6dc44b363b7383dcd4fe., hostname=bd6592c37bb2,40087,1732486423713, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8ca644d2cada6dc44b363b7383dcd4fe, server=bd6592c37bb2,40087,1732486423713 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T22:16:32,245 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732486585339.8ca644d2cada6dc44b363b7383dcd4fe., hostname=bd6592c37bb2,40087,1732486423713, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8ca644d2cada6dc44b363b7383dcd4fe, server=bd6592c37bb2,40087,1732486423713 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T22:16:32,245 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732486585339.8ca644d2cada6dc44b363b7383dcd4fe., hostname=bd6592c37bb2,40087,1732486423713, seqNum=5 because the exception is null or not the one we care about 2024-11-24T22:16:32,255 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T22:16:32,258 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241124f9c897fa992444198f55db8a31246739_8ca644d2cada6dc44b363b7383dcd4fe to hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241124f9c897fa992444198f55db8a31246739_8ca644d2cada6dc44b363b7383dcd4fe 2024-11-24T22:16:32,259 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/8ca644d2cada6dc44b363b7383dcd4fe/.tmp/A/ba62e5cf94804332b0c6667d4633e13d, store: [table=TestAcidGuarantees family=A region=8ca644d2cada6dc44b363b7383dcd4fe] 2024-11-24T22:16:32,260 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/8ca644d2cada6dc44b363b7383dcd4fe/.tmp/A/ba62e5cf94804332b0c6667d4633e13d is 175, key is test_row_0/A:col10/1732486591833/Put/seqid=0 2024-11-24T22:16:32,263 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45929 is added to blk_1073742395_1571 (size=30951) 2024-11-24T22:16:32,504 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40087 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8ca644d2cada6dc44b363b7383dcd4fe, server=bd6592c37bb2,40087,1732486423713 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-24T22:16:32,504 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40087 {}] ipc.CallRunner(138): callId: 29 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49026 deadline: 1732486652501, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8ca644d2cada6dc44b363b7383dcd4fe, server=bd6592c37bb2,40087,1732486423713 2024-11-24T22:16:32,504 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40087 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8ca644d2cada6dc44b363b7383dcd4fe, server=bd6592c37bb2,40087,1732486423713 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-24T22:16:32,504 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40087 {}] ipc.CallRunner(138): callId: 24 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49060 deadline: 1732486652501, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8ca644d2cada6dc44b363b7383dcd4fe, server=bd6592c37bb2,40087,1732486423713 2024-11-24T22:16:32,504 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732486585339.8ca644d2cada6dc44b363b7383dcd4fe., hostname=bd6592c37bb2,40087,1732486423713, seqNum=5 , the old value is region=TestAcidGuarantees,,1732486585339.8ca644d2cada6dc44b363b7383dcd4fe., hostname=bd6592c37bb2,40087,1732486423713, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8ca644d2cada6dc44b363b7383dcd4fe, server=bd6592c37bb2,40087,1732486423713 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T22:16:32,504 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732486585339.8ca644d2cada6dc44b363b7383dcd4fe., hostname=bd6592c37bb2,40087,1732486423713, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8ca644d2cada6dc44b363b7383dcd4fe, server=bd6592c37bb2,40087,1732486423713 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T22:16:32,504 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732486585339.8ca644d2cada6dc44b363b7383dcd4fe., hostname=bd6592c37bb2,40087,1732486423713, seqNum=5 because the exception is null or not the one we care about 2024-11-24T22:16:32,504 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732486585339.8ca644d2cada6dc44b363b7383dcd4fe., hostname=bd6592c37bb2,40087,1732486423713, seqNum=5 , the old value is region=TestAcidGuarantees,,1732486585339.8ca644d2cada6dc44b363b7383dcd4fe., hostname=bd6592c37bb2,40087,1732486423713, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8ca644d2cada6dc44b363b7383dcd4fe, server=bd6592c37bb2,40087,1732486423713 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T22:16:32,504 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732486585339.8ca644d2cada6dc44b363b7383dcd4fe., hostname=bd6592c37bb2,40087,1732486423713, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8ca644d2cada6dc44b363b7383dcd4fe, server=bd6592c37bb2,40087,1732486423713 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T22:16:32,505 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732486585339.8ca644d2cada6dc44b363b7383dcd4fe., hostname=bd6592c37bb2,40087,1732486423713, seqNum=5 because the exception is null or not the one we care about 2024-11-24T22:16:32,554 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40087 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8ca644d2cada6dc44b363b7383dcd4fe, server=bd6592c37bb2,40087,1732486423713 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-24T22:16:32,554 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40087 {}] ipc.CallRunner(138): callId: 38 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48986 deadline: 1732486652551, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8ca644d2cada6dc44b363b7383dcd4fe, server=bd6592c37bb2,40087,1732486423713 2024-11-24T22:16:32,554 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40087 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8ca644d2cada6dc44b363b7383dcd4fe, server=bd6592c37bb2,40087,1732486423713 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-24T22:16:32,554 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40087 {}] ipc.CallRunner(138): callId: 34 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49002 deadline: 1732486652551, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8ca644d2cada6dc44b363b7383dcd4fe, server=bd6592c37bb2,40087,1732486423713 2024-11-24T22:16:32,555 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732486585339.8ca644d2cada6dc44b363b7383dcd4fe., hostname=bd6592c37bb2,40087,1732486423713, seqNum=5 , the old value is region=TestAcidGuarantees,,1732486585339.8ca644d2cada6dc44b363b7383dcd4fe., hostname=bd6592c37bb2,40087,1732486423713, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8ca644d2cada6dc44b363b7383dcd4fe, server=bd6592c37bb2,40087,1732486423713 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T22:16:32,555 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732486585339.8ca644d2cada6dc44b363b7383dcd4fe., hostname=bd6592c37bb2,40087,1732486423713, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8ca644d2cada6dc44b363b7383dcd4fe, server=bd6592c37bb2,40087,1732486423713 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T22:16:32,555 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732486585339.8ca644d2cada6dc44b363b7383dcd4fe., hostname=bd6592c37bb2,40087,1732486423713, seqNum=5 , the old value is region=TestAcidGuarantees,,1732486585339.8ca644d2cada6dc44b363b7383dcd4fe., hostname=bd6592c37bb2,40087,1732486423713, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8ca644d2cada6dc44b363b7383dcd4fe, server=bd6592c37bb2,40087,1732486423713 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T22:16:32,555 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732486585339.8ca644d2cada6dc44b363b7383dcd4fe., hostname=bd6592c37bb2,40087,1732486423713, seqNum=5 because the exception is null or not the one we care about 2024-11-24T22:16:32,555 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732486585339.8ca644d2cada6dc44b363b7383dcd4fe., hostname=bd6592c37bb2,40087,1732486423713, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8ca644d2cada6dc44b363b7383dcd4fe, server=bd6592c37bb2,40087,1732486423713 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T22:16:32,555 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732486585339.8ca644d2cada6dc44b363b7383dcd4fe., hostname=bd6592c37bb2,40087,1732486423713, seqNum=5 because the exception is null or not the one we care about 2024-11-24T22:16:32,600 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37701 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=152 2024-11-24T22:16:32,601 INFO [RPCClient-NioEventLoopGroup-4-12 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-24T22:16:32,601 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37701 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-24T22:16:32,602 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37701 {}] procedure2.ProcedureExecutor(1139): Stored pid=154, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=154, table=TestAcidGuarantees 2024-11-24T22:16:32,603 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37701 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=154 2024-11-24T22:16:32,603 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=154, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=154, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-24T22:16:32,603 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=154, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=154, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-24T22:16:32,604 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=155, ppid=154, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-24T22:16:32,663 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=61, memsize=20.1 K, hasBloomFilter=true, into tmp file hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/8ca644d2cada6dc44b363b7383dcd4fe/.tmp/A/ba62e5cf94804332b0c6667d4633e13d 2024-11-24T22:16:32,670 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/8ca644d2cada6dc44b363b7383dcd4fe/.tmp/B/36dd327c79934afa8769997291f671da is 50, key is test_row_0/B:col10/1732486591833/Put/seqid=0 2024-11-24T22:16:32,673 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45929 is added to blk_1073742396_1572 (size=9657) 2024-11-24T22:16:32,673 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=61 (bloomFilter=true), to=hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/8ca644d2cada6dc44b363b7383dcd4fe/.tmp/B/36dd327c79934afa8769997291f671da 2024-11-24T22:16:32,679 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/8ca644d2cada6dc44b363b7383dcd4fe/.tmp/C/4739455ff5084251ac728987a900105a is 50, key is test_row_0/C:col10/1732486591833/Put/seqid=0 2024-11-24T22:16:32,682 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45929 is added to blk_1073742397_1573 (size=9657) 2024-11-24T22:16:32,710 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37701 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=154 2024-11-24T22:16:32,721 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40087 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8ca644d2cada6dc44b363b7383dcd4fe, server=bd6592c37bb2,40087,1732486423713 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-24T22:16:32,722 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40087 {}] ipc.CallRunner(138): callId: 14 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49050 deadline: 1732486652720, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8ca644d2cada6dc44b363b7383dcd4fe, server=bd6592c37bb2,40087,1732486423713 2024-11-24T22:16:32,722 WARN [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRpcRetryingCaller(177): Call to bd6592c37bb2,40087,1732486423713 for 'test_row_0' in 8ca644d2cada6dc44b363b7383dcd4fe of TestAcidGuarantees failed, tries = 7, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 4189 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8ca644d2cada6dc44b363b7383dcd4fe, server=bd6592c37bb2,40087,1732486423713 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor44.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8ca644d2cada6dc44b363b7383dcd4fe, server=bd6592c37bb2,40087,1732486423713 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-24T22:16:32,723 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732486585339.8ca644d2cada6dc44b363b7383dcd4fe., hostname=bd6592c37bb2,40087,1732486423713, seqNum=5 , the old value is region=TestAcidGuarantees,,1732486585339.8ca644d2cada6dc44b363b7383dcd4fe., hostname=bd6592c37bb2,40087,1732486423713, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8ca644d2cada6dc44b363b7383dcd4fe, server=bd6592c37bb2,40087,1732486423713 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T22:16:32,723 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732486585339.8ca644d2cada6dc44b363b7383dcd4fe., hostname=bd6592c37bb2,40087,1732486423713, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8ca644d2cada6dc44b363b7383dcd4fe, server=bd6592c37bb2,40087,1732486423713 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T22:16:32,723 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732486585339.8ca644d2cada6dc44b363b7383dcd4fe., hostname=bd6592c37bb2,40087,1732486423713, seqNum=5 because the exception is null or not the one we care about 2024-11-24T22:16:32,755 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=40087 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=155 2024-11-24T22:16:32,755 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bd6592c37bb2:0-0 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732486585339.8ca644d2cada6dc44b363b7383dcd4fe. 2024-11-24T22:16:32,756 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bd6592c37bb2:0-0 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732486585339.8ca644d2cada6dc44b363b7383dcd4fe. as already flushing 2024-11-24T22:16:32,756 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bd6592c37bb2:0-0 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732486585339.8ca644d2cada6dc44b363b7383dcd4fe. 2024-11-24T22:16:32,756 ERROR [RS_FLUSH_OPERATIONS-regionserver/bd6592c37bb2:0-0 {event_type=RS_FLUSH_REGIONS, pid=155}] handler.RSProcedureHandler(58): pid=155 java.io.IOException: Unable to complete flush {ENCODED => 8ca644d2cada6dc44b363b7383dcd4fe, NAME => 'TestAcidGuarantees,,1732486585339.8ca644d2cada6dc44b363b7383dcd4fe.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T22:16:32,756 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bd6592c37bb2:0-0 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=155 java.io.IOException: Unable to complete flush {ENCODED => 8ca644d2cada6dc44b363b7383dcd4fe, NAME => 'TestAcidGuarantees,,1732486585339.8ca644d2cada6dc44b363b7383dcd4fe.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T22:16:32,756 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37701 {}] master.HMaster(4177): Remote procedure failed, pid=155 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 8ca644d2cada6dc44b363b7383dcd4fe, NAME => 'TestAcidGuarantees,,1732486585339.8ca644d2cada6dc44b363b7383dcd4fe.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 8ca644d2cada6dc44b363b7383dcd4fe, NAME => 'TestAcidGuarantees,,1732486585339.8ca644d2cada6dc44b363b7383dcd4fe.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T22:16:32,908 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=40087 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=155 2024-11-24T22:16:32,908 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bd6592c37bb2:0-1 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732486585339.8ca644d2cada6dc44b363b7383dcd4fe. 2024-11-24T22:16:32,908 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bd6592c37bb2:0-1 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732486585339.8ca644d2cada6dc44b363b7383dcd4fe. as already flushing 2024-11-24T22:16:32,908 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bd6592c37bb2:0-1 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732486585339.8ca644d2cada6dc44b363b7383dcd4fe. 2024-11-24T22:16:32,908 ERROR [RS_FLUSH_OPERATIONS-regionserver/bd6592c37bb2:0-1 {event_type=RS_FLUSH_REGIONS, pid=155}] handler.RSProcedureHandler(58): pid=155 java.io.IOException: Unable to complete flush {ENCODED => 8ca644d2cada6dc44b363b7383dcd4fe, NAME => 'TestAcidGuarantees,,1732486585339.8ca644d2cada6dc44b363b7383dcd4fe.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T22:16:32,909 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bd6592c37bb2:0-1 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=155 java.io.IOException: Unable to complete flush {ENCODED => 8ca644d2cada6dc44b363b7383dcd4fe, NAME => 'TestAcidGuarantees,,1732486585339.8ca644d2cada6dc44b363b7383dcd4fe.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T22:16:32,909 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37701 {}] master.HMaster(4177): Remote procedure failed, pid=155 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 8ca644d2cada6dc44b363b7383dcd4fe, NAME => 'TestAcidGuarantees,,1732486585339.8ca644d2cada6dc44b363b7383dcd4fe.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 8ca644d2cada6dc44b363b7383dcd4fe, NAME => 'TestAcidGuarantees,,1732486585339.8ca644d2cada6dc44b363b7383dcd4fe.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T22:16:32,921 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37701 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=154 2024-11-24T22:16:33,013 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40087 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8ca644d2cada6dc44b363b7383dcd4fe, server=bd6592c37bb2,40087,1732486423713 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-24T22:16:33,014 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40087 {}] ipc.CallRunner(138): callId: 30 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49026 deadline: 1732486653011, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8ca644d2cada6dc44b363b7383dcd4fe, server=bd6592c37bb2,40087,1732486423713 2024-11-24T22:16:33,014 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40087 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8ca644d2cada6dc44b363b7383dcd4fe, server=bd6592c37bb2,40087,1732486423713 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-24T22:16:33,014 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40087 {}] ipc.CallRunner(138): callId: 25 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49060 deadline: 1732486653011, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8ca644d2cada6dc44b363b7383dcd4fe, server=bd6592c37bb2,40087,1732486423713 2024-11-24T22:16:33,014 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732486585339.8ca644d2cada6dc44b363b7383dcd4fe., hostname=bd6592c37bb2,40087,1732486423713, seqNum=5 , the old value is region=TestAcidGuarantees,,1732486585339.8ca644d2cada6dc44b363b7383dcd4fe., hostname=bd6592c37bb2,40087,1732486423713, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8ca644d2cada6dc44b363b7383dcd4fe, server=bd6592c37bb2,40087,1732486423713 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T22:16:33,014 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732486585339.8ca644d2cada6dc44b363b7383dcd4fe., hostname=bd6592c37bb2,40087,1732486423713, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8ca644d2cada6dc44b363b7383dcd4fe, server=bd6592c37bb2,40087,1732486423713 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T22:16:33,014 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732486585339.8ca644d2cada6dc44b363b7383dcd4fe., hostname=bd6592c37bb2,40087,1732486423713, seqNum=5 because the exception is null or not the one we care about 2024-11-24T22:16:33,014 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732486585339.8ca644d2cada6dc44b363b7383dcd4fe., hostname=bd6592c37bb2,40087,1732486423713, seqNum=5 , the old value is region=TestAcidGuarantees,,1732486585339.8ca644d2cada6dc44b363b7383dcd4fe., hostname=bd6592c37bb2,40087,1732486423713, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8ca644d2cada6dc44b363b7383dcd4fe, server=bd6592c37bb2,40087,1732486423713 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T22:16:33,014 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732486585339.8ca644d2cada6dc44b363b7383dcd4fe., hostname=bd6592c37bb2,40087,1732486423713, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8ca644d2cada6dc44b363b7383dcd4fe, server=bd6592c37bb2,40087,1732486423713 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T22:16:33,014 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732486585339.8ca644d2cada6dc44b363b7383dcd4fe., hostname=bd6592c37bb2,40087,1732486423713, seqNum=5 because the exception is null or not the one we care about 2024-11-24T22:16:33,061 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=40087 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=155 2024-11-24T22:16:33,061 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bd6592c37bb2:0-2 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732486585339.8ca644d2cada6dc44b363b7383dcd4fe. 2024-11-24T22:16:33,061 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bd6592c37bb2:0-2 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732486585339.8ca644d2cada6dc44b363b7383dcd4fe. as already flushing 2024-11-24T22:16:33,061 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bd6592c37bb2:0-2 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732486585339.8ca644d2cada6dc44b363b7383dcd4fe. 2024-11-24T22:16:33,061 ERROR [RS_FLUSH_OPERATIONS-regionserver/bd6592c37bb2:0-2 {event_type=RS_FLUSH_REGIONS, pid=155}] handler.RSProcedureHandler(58): pid=155 java.io.IOException: Unable to complete flush {ENCODED => 8ca644d2cada6dc44b363b7383dcd4fe, NAME => 'TestAcidGuarantees,,1732486585339.8ca644d2cada6dc44b363b7383dcd4fe.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T22:16:33,061 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bd6592c37bb2:0-2 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=155 java.io.IOException: Unable to complete flush {ENCODED => 8ca644d2cada6dc44b363b7383dcd4fe, NAME => 'TestAcidGuarantees,,1732486585339.8ca644d2cada6dc44b363b7383dcd4fe.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T22:16:33,062 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37701 {}] master.HMaster(4177): Remote procedure failed, pid=155 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 8ca644d2cada6dc44b363b7383dcd4fe, NAME => 'TestAcidGuarantees,,1732486585339.8ca644d2cada6dc44b363b7383dcd4fe.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 8ca644d2cada6dc44b363b7383dcd4fe, NAME => 'TestAcidGuarantees,,1732486585339.8ca644d2cada6dc44b363b7383dcd4fe.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T22:16:33,065 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40087 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8ca644d2cada6dc44b363b7383dcd4fe, server=bd6592c37bb2,40087,1732486423713 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-24T22:16:33,065 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40087 {}] ipc.CallRunner(138): callId: 39 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48986 deadline: 1732486653060, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8ca644d2cada6dc44b363b7383dcd4fe, server=bd6592c37bb2,40087,1732486423713 2024-11-24T22:16:33,066 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40087 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8ca644d2cada6dc44b363b7383dcd4fe, server=bd6592c37bb2,40087,1732486423713 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-24T22:16:33,066 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732486585339.8ca644d2cada6dc44b363b7383dcd4fe., hostname=bd6592c37bb2,40087,1732486423713, seqNum=5 , the old value is region=TestAcidGuarantees,,1732486585339.8ca644d2cada6dc44b363b7383dcd4fe., hostname=bd6592c37bb2,40087,1732486423713, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8ca644d2cada6dc44b363b7383dcd4fe, server=bd6592c37bb2,40087,1732486423713 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T22:16:33,066 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732486585339.8ca644d2cada6dc44b363b7383dcd4fe., hostname=bd6592c37bb2,40087,1732486423713, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8ca644d2cada6dc44b363b7383dcd4fe, server=bd6592c37bb2,40087,1732486423713 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T22:16:33,066 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732486585339.8ca644d2cada6dc44b363b7383dcd4fe., hostname=bd6592c37bb2,40087,1732486423713, seqNum=5 because the exception is null or not the one we care about 2024-11-24T22:16:33,066 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40087 {}] ipc.CallRunner(138): callId: 35 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49002 deadline: 1732486653060, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8ca644d2cada6dc44b363b7383dcd4fe, server=bd6592c37bb2,40087,1732486423713 2024-11-24T22:16:33,066 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732486585339.8ca644d2cada6dc44b363b7383dcd4fe., hostname=bd6592c37bb2,40087,1732486423713, seqNum=5 , the old value is region=TestAcidGuarantees,,1732486585339.8ca644d2cada6dc44b363b7383dcd4fe., hostname=bd6592c37bb2,40087,1732486423713, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8ca644d2cada6dc44b363b7383dcd4fe, server=bd6592c37bb2,40087,1732486423713 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T22:16:33,066 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732486585339.8ca644d2cada6dc44b363b7383dcd4fe., hostname=bd6592c37bb2,40087,1732486423713, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8ca644d2cada6dc44b363b7383dcd4fe, server=bd6592c37bb2,40087,1732486423713 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T22:16:33,066 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732486585339.8ca644d2cada6dc44b363b7383dcd4fe., hostname=bd6592c37bb2,40087,1732486423713, seqNum=5 because the exception is null or not the one we care about 2024-11-24T22:16:33,083 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=61 (bloomFilter=true), to=hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/8ca644d2cada6dc44b363b7383dcd4fe/.tmp/C/4739455ff5084251ac728987a900105a 2024-11-24T22:16:33,086 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/8ca644d2cada6dc44b363b7383dcd4fe/.tmp/A/ba62e5cf94804332b0c6667d4633e13d as hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/8ca644d2cada6dc44b363b7383dcd4fe/A/ba62e5cf94804332b0c6667d4633e13d 2024-11-24T22:16:33,089 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/8ca644d2cada6dc44b363b7383dcd4fe/A/ba62e5cf94804332b0c6667d4633e13d, entries=150, sequenceid=61, filesize=30.2 K 2024-11-24T22:16:33,090 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/8ca644d2cada6dc44b363b7383dcd4fe/.tmp/B/36dd327c79934afa8769997291f671da as hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/8ca644d2cada6dc44b363b7383dcd4fe/B/36dd327c79934afa8769997291f671da 2024-11-24T22:16:33,093 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/8ca644d2cada6dc44b363b7383dcd4fe/B/36dd327c79934afa8769997291f671da, entries=100, sequenceid=61, filesize=9.4 K 2024-11-24T22:16:33,093 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/8ca644d2cada6dc44b363b7383dcd4fe/.tmp/C/4739455ff5084251ac728987a900105a as hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/8ca644d2cada6dc44b363b7383dcd4fe/C/4739455ff5084251ac728987a900105a 2024-11-24T22:16:33,098 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/8ca644d2cada6dc44b363b7383dcd4fe/C/4739455ff5084251ac728987a900105a, entries=100, sequenceid=61, filesize=9.4 K 2024-11-24T22:16:33,099 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~60.38 KB/61830, heapSize ~180 KB/184320, currentSize=120.76 KB/123660 for 8ca644d2cada6dc44b363b7383dcd4fe in 1264ms, sequenceid=61, compaction requested=true 2024-11-24T22:16:33,099 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 8ca644d2cada6dc44b363b7383dcd4fe: 2024-11-24T22:16:33,099 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 8ca644d2cada6dc44b363b7383dcd4fe:A, priority=-2147483648, current under compaction store size is 1 2024-11-24T22:16:33,099 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-24T22:16:33,099 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 8ca644d2cada6dc44b363b7383dcd4fe:B, priority=-2147483648, current under compaction store size is 2 2024-11-24T22:16:33,099 DEBUG [RS:0;bd6592c37bb2:40087-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-24T22:16:33,099 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-24T22:16:33,099 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 8ca644d2cada6dc44b363b7383dcd4fe:C, priority=-2147483648, current under compaction store size is 3 2024-11-24T22:16:33,099 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-24T22:16:33,099 DEBUG [RS:0;bd6592c37bb2:40087-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-24T22:16:33,100 DEBUG [RS:0;bd6592c37bb2:40087-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 33659 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-24T22:16:33,100 DEBUG [RS:0;bd6592c37bb2:40087-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 92861 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-24T22:16:33,100 DEBUG [RS:0;bd6592c37bb2:40087-shortCompactions-0 {}] regionserver.HStore(1541): 8ca644d2cada6dc44b363b7383dcd4fe/A is initiating minor compaction (all files) 2024-11-24T22:16:33,100 DEBUG [RS:0;bd6592c37bb2:40087-longCompactions-0 {}] regionserver.HStore(1541): 8ca644d2cada6dc44b363b7383dcd4fe/B is initiating minor compaction (all files) 2024-11-24T22:16:33,100 INFO [RS:0;bd6592c37bb2:40087-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 8ca644d2cada6dc44b363b7383dcd4fe/A in TestAcidGuarantees,,1732486585339.8ca644d2cada6dc44b363b7383dcd4fe. 2024-11-24T22:16:33,100 INFO [RS:0;bd6592c37bb2:40087-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 8ca644d2cada6dc44b363b7383dcd4fe/B in TestAcidGuarantees,,1732486585339.8ca644d2cada6dc44b363b7383dcd4fe. 2024-11-24T22:16:33,100 INFO [RS:0;bd6592c37bb2:40087-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/8ca644d2cada6dc44b363b7383dcd4fe/A/4d8ef082bbc741109521c9a43715f049, hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/8ca644d2cada6dc44b363b7383dcd4fe/A/35a85100955447888670863594d84360, hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/8ca644d2cada6dc44b363b7383dcd4fe/A/ba62e5cf94804332b0c6667d4633e13d] into tmpdir=hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/8ca644d2cada6dc44b363b7383dcd4fe/.tmp, totalSize=90.7 K 2024-11-24T22:16:33,100 INFO [RS:0;bd6592c37bb2:40087-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/8ca644d2cada6dc44b363b7383dcd4fe/B/be99041ff522499f83a39a00787ed3ce, hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/8ca644d2cada6dc44b363b7383dcd4fe/B/c807ab7919a94e47a5f40b2f71d3a197, hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/8ca644d2cada6dc44b363b7383dcd4fe/B/36dd327c79934afa8769997291f671da] into tmpdir=hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/8ca644d2cada6dc44b363b7383dcd4fe/.tmp, totalSize=32.9 K 2024-11-24T22:16:33,100 INFO [RS:0;bd6592c37bb2:40087-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=13 throughput controller=DefaultCompactionThroughputController [maxThroughput=57.69 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1732486585339.8ca644d2cada6dc44b363b7383dcd4fe. 2024-11-24T22:16:33,100 DEBUG [RS:0;bd6592c37bb2:40087-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1732486585339.8ca644d2cada6dc44b363b7383dcd4fe. files: [hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/8ca644d2cada6dc44b363b7383dcd4fe/A/4d8ef082bbc741109521c9a43715f049, hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/8ca644d2cada6dc44b363b7383dcd4fe/A/35a85100955447888670863594d84360, hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/8ca644d2cada6dc44b363b7383dcd4fe/A/ba62e5cf94804332b0c6667d4633e13d] 2024-11-24T22:16:33,100 DEBUG [RS:0;bd6592c37bb2:40087-longCompactions-0 {}] compactions.Compactor(225): Compacting be99041ff522499f83a39a00787ed3ce, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=14, earliestPutTs=1732486588443 2024-11-24T22:16:33,100 DEBUG [RS:0;bd6592c37bb2:40087-shortCompactions-0 {}] compactions.Compactor(225): Compacting 4d8ef082bbc741109521c9a43715f049, keycount=150, bloomtype=ROW, size=30.2 K, encoding=NONE, compression=NONE, seqNum=14, earliestPutTs=1732486588443 2024-11-24T22:16:33,101 DEBUG [RS:0;bd6592c37bb2:40087-longCompactions-0 {}] compactions.Compactor(225): Compacting c807ab7919a94e47a5f40b2f71d3a197, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=49, earliestPutTs=1732486588524 2024-11-24T22:16:33,101 DEBUG [RS:0;bd6592c37bb2:40087-shortCompactions-0 {}] compactions.Compactor(225): Compacting 35a85100955447888670863594d84360, keycount=150, bloomtype=ROW, size=30.2 K, encoding=NONE, compression=NONE, seqNum=49, earliestPutTs=1732486588524 2024-11-24T22:16:33,101 DEBUG [RS:0;bd6592c37bb2:40087-longCompactions-0 {}] compactions.Compactor(225): Compacting 36dd327c79934afa8769997291f671da, keycount=100, bloomtype=ROW, size=9.4 K, encoding=NONE, compression=NONE, seqNum=61, earliestPutTs=1732486590696 2024-11-24T22:16:33,101 DEBUG [RS:0;bd6592c37bb2:40087-shortCompactions-0 {}] compactions.Compactor(225): Compacting ba62e5cf94804332b0c6667d4633e13d, keycount=150, bloomtype=ROW, size=30.2 K, encoding=NONE, compression=NONE, seqNum=61, earliestPutTs=1732486590696 2024-11-24T22:16:33,106 INFO [RS:0;bd6592c37bb2:40087-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=8ca644d2cada6dc44b363b7383dcd4fe] 2024-11-24T22:16:33,106 INFO [RS:0;bd6592c37bb2:40087-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 8ca644d2cada6dc44b363b7383dcd4fe#B#compaction#488 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 57.69 MB/second 2024-11-24T22:16:33,107 DEBUG [RS:0;bd6592c37bb2:40087-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/8ca644d2cada6dc44b363b7383dcd4fe/.tmp/B/9e0fd71ad3754b308cad5b3311354f00 is 50, key is test_row_0/B:col10/1732486591833/Put/seqid=0 2024-11-24T22:16:33,108 DEBUG [RS:0;bd6592c37bb2:40087-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e202411240a84917d133e4524814e67b6129ca904_8ca644d2cada6dc44b363b7383dcd4fe store=[table=TestAcidGuarantees family=A region=8ca644d2cada6dc44b363b7383dcd4fe] 2024-11-24T22:16:33,109 DEBUG [RS:0;bd6592c37bb2:40087-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e202411240a84917d133e4524814e67b6129ca904_8ca644d2cada6dc44b363b7383dcd4fe, store=[table=TestAcidGuarantees family=A region=8ca644d2cada6dc44b363b7383dcd4fe] 2024-11-24T22:16:33,109 DEBUG [RS:0;bd6592c37bb2:40087-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411240a84917d133e4524814e67b6129ca904_8ca644d2cada6dc44b363b7383dcd4fe because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=8ca644d2cada6dc44b363b7383dcd4fe] 2024-11-24T22:16:33,112 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45929 is added to blk_1073742398_1574 (size=12104) 2024-11-24T22:16:33,113 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45929 is added to blk_1073742399_1575 (size=4469) 2024-11-24T22:16:33,113 INFO [RS:0;bd6592c37bb2:40087-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 8ca644d2cada6dc44b363b7383dcd4fe#A#compaction#489 average throughput is 3.49 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 57.69 MB/second 2024-11-24T22:16:33,114 DEBUG [RS:0;bd6592c37bb2:40087-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/8ca644d2cada6dc44b363b7383dcd4fe/.tmp/A/ff60faeb807c4decab1b2a1f7ba31d85 is 175, key is test_row_0/A:col10/1732486591833/Put/seqid=0 2024-11-24T22:16:33,124 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45929 is added to blk_1073742400_1576 (size=31165) 2024-11-24T22:16:33,214 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=40087 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=155 2024-11-24T22:16:33,214 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bd6592c37bb2:0-0 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732486585339.8ca644d2cada6dc44b363b7383dcd4fe. 2024-11-24T22:16:33,214 INFO [RS_FLUSH_OPERATIONS-regionserver/bd6592c37bb2:0-0 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.HRegion(2902): Flushing 8ca644d2cada6dc44b363b7383dcd4fe 3/3 column families, dataSize=120.76 KB heapSize=345.14 KB 2024-11-24T22:16:33,214 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bd6592c37bb2:0-0 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 8ca644d2cada6dc44b363b7383dcd4fe, store=A 2024-11-24T22:16:33,214 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bd6592c37bb2:0-0 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-24T22:16:33,214 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bd6592c37bb2:0-0 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 8ca644d2cada6dc44b363b7383dcd4fe, store=B 2024-11-24T22:16:33,214 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bd6592c37bb2:0-0 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-24T22:16:33,214 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bd6592c37bb2:0-0 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 8ca644d2cada6dc44b363b7383dcd4fe, store=C 2024-11-24T22:16:33,214 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bd6592c37bb2:0-0 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-24T22:16:33,221 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bd6592c37bb2:0-0 {event_type=RS_FLUSH_REGIONS, pid=155}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241124e0bdee4dd60c465bbb70df873731e23d_8ca644d2cada6dc44b363b7383dcd4fe is 50, key is test_row_0/A:col10/1732486591911/Put/seqid=0 2024-11-24T22:16:33,224 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45929 is added to blk_1073742401_1577 (size=12154) 2024-11-24T22:16:33,231 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37701 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=154 2024-11-24T22:16:33,516 DEBUG [RS:0;bd6592c37bb2:40087-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/8ca644d2cada6dc44b363b7383dcd4fe/.tmp/B/9e0fd71ad3754b308cad5b3311354f00 as hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/8ca644d2cada6dc44b363b7383dcd4fe/B/9e0fd71ad3754b308cad5b3311354f00 2024-11-24T22:16:33,521 INFO [RS:0;bd6592c37bb2:40087-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 8ca644d2cada6dc44b363b7383dcd4fe/B of 8ca644d2cada6dc44b363b7383dcd4fe into 9e0fd71ad3754b308cad5b3311354f00(size=11.8 K), total size for store is 11.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-24T22:16:33,521 DEBUG [RS:0;bd6592c37bb2:40087-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 8ca644d2cada6dc44b363b7383dcd4fe: 2024-11-24T22:16:33,521 INFO [RS:0;bd6592c37bb2:40087-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732486585339.8ca644d2cada6dc44b363b7383dcd4fe., storeName=8ca644d2cada6dc44b363b7383dcd4fe/B, priority=13, startTime=1732486593099; duration=0sec 2024-11-24T22:16:33,521 DEBUG [RS:0;bd6592c37bb2:40087-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-24T22:16:33,521 DEBUG [RS:0;bd6592c37bb2:40087-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 8ca644d2cada6dc44b363b7383dcd4fe:B 2024-11-24T22:16:33,521 DEBUG [RS:0;bd6592c37bb2:40087-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-24T22:16:33,522 DEBUG [RS:0;bd6592c37bb2:40087-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 33659 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-24T22:16:33,522 DEBUG [RS:0;bd6592c37bb2:40087-longCompactions-0 {}] regionserver.HStore(1541): 8ca644d2cada6dc44b363b7383dcd4fe/C is initiating minor compaction (all files) 2024-11-24T22:16:33,522 INFO [RS:0;bd6592c37bb2:40087-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 8ca644d2cada6dc44b363b7383dcd4fe/C in TestAcidGuarantees,,1732486585339.8ca644d2cada6dc44b363b7383dcd4fe. 2024-11-24T22:16:33,522 INFO [RS:0;bd6592c37bb2:40087-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/8ca644d2cada6dc44b363b7383dcd4fe/C/073e5ad8128f4a9fa041c49b3dd915d6, hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/8ca644d2cada6dc44b363b7383dcd4fe/C/63f9798f78214b7ab82aae257603936a, hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/8ca644d2cada6dc44b363b7383dcd4fe/C/4739455ff5084251ac728987a900105a] into tmpdir=hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/8ca644d2cada6dc44b363b7383dcd4fe/.tmp, totalSize=32.9 K 2024-11-24T22:16:33,523 DEBUG [RS:0;bd6592c37bb2:40087-longCompactions-0 {}] compactions.Compactor(225): Compacting 073e5ad8128f4a9fa041c49b3dd915d6, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=14, earliestPutTs=1732486588443 2024-11-24T22:16:33,523 DEBUG [RS:0;bd6592c37bb2:40087-longCompactions-0 {}] compactions.Compactor(225): Compacting 63f9798f78214b7ab82aae257603936a, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=49, earliestPutTs=1732486588524 2024-11-24T22:16:33,523 DEBUG [RS:0;bd6592c37bb2:40087-longCompactions-0 {}] compactions.Compactor(225): Compacting 4739455ff5084251ac728987a900105a, keycount=100, bloomtype=ROW, size=9.4 K, encoding=NONE, compression=NONE, seqNum=61, earliestPutTs=1732486590696 2024-11-24T22:16:33,526 DEBUG [RS:0;bd6592c37bb2:40087-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/8ca644d2cada6dc44b363b7383dcd4fe/.tmp/A/ff60faeb807c4decab1b2a1f7ba31d85 as hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/8ca644d2cada6dc44b363b7383dcd4fe/A/ff60faeb807c4decab1b2a1f7ba31d85 2024-11-24T22:16:33,530 INFO [RS:0;bd6592c37bb2:40087-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 8ca644d2cada6dc44b363b7383dcd4fe/A of 8ca644d2cada6dc44b363b7383dcd4fe into ff60faeb807c4decab1b2a1f7ba31d85(size=30.4 K), total size for store is 30.4 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-24T22:16:33,530 DEBUG [RS:0;bd6592c37bb2:40087-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 8ca644d2cada6dc44b363b7383dcd4fe: 2024-11-24T22:16:33,530 INFO [RS:0;bd6592c37bb2:40087-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732486585339.8ca644d2cada6dc44b363b7383dcd4fe., storeName=8ca644d2cada6dc44b363b7383dcd4fe/A, priority=13, startTime=1732486593099; duration=0sec 2024-11-24T22:16:33,530 DEBUG [RS:0;bd6592c37bb2:40087-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-24T22:16:33,530 DEBUG [RS:0;bd6592c37bb2:40087-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 8ca644d2cada6dc44b363b7383dcd4fe:A 2024-11-24T22:16:33,531 INFO [RS:0;bd6592c37bb2:40087-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 8ca644d2cada6dc44b363b7383dcd4fe#C#compaction#491 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 57.69 MB/second 2024-11-24T22:16:33,531 DEBUG [RS:0;bd6592c37bb2:40087-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/8ca644d2cada6dc44b363b7383dcd4fe/.tmp/C/77cfe1f3e45a4ae4a303695fd6b77f4a is 50, key is test_row_0/C:col10/1732486591833/Put/seqid=0 2024-11-24T22:16:33,538 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45929 is added to blk_1073742402_1578 (size=12104) 2024-11-24T22:16:33,543 DEBUG [RS:0;bd6592c37bb2:40087-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/8ca644d2cada6dc44b363b7383dcd4fe/.tmp/C/77cfe1f3e45a4ae4a303695fd6b77f4a as hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/8ca644d2cada6dc44b363b7383dcd4fe/C/77cfe1f3e45a4ae4a303695fd6b77f4a 2024-11-24T22:16:33,547 INFO [RS:0;bd6592c37bb2:40087-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 8ca644d2cada6dc44b363b7383dcd4fe/C of 8ca644d2cada6dc44b363b7383dcd4fe into 77cfe1f3e45a4ae4a303695fd6b77f4a(size=11.8 K), total size for store is 11.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-24T22:16:33,547 DEBUG [RS:0;bd6592c37bb2:40087-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 8ca644d2cada6dc44b363b7383dcd4fe: 2024-11-24T22:16:33,547 INFO [RS:0;bd6592c37bb2:40087-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732486585339.8ca644d2cada6dc44b363b7383dcd4fe., storeName=8ca644d2cada6dc44b363b7383dcd4fe/C, priority=13, startTime=1732486593099; duration=0sec 2024-11-24T22:16:33,547 DEBUG [RS:0;bd6592c37bb2:40087-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-24T22:16:33,547 DEBUG [RS:0;bd6592c37bb2:40087-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 8ca644d2cada6dc44b363b7383dcd4fe:C 2024-11-24T22:16:33,624 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bd6592c37bb2:0-0 {event_type=RS_FLUSH_REGIONS, pid=155}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T22:16:33,627 INFO [RS_FLUSH_OPERATIONS-regionserver/bd6592c37bb2:0-0 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241124e0bdee4dd60c465bbb70df873731e23d_8ca644d2cada6dc44b363b7383dcd4fe to hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241124e0bdee4dd60c465bbb70df873731e23d_8ca644d2cada6dc44b363b7383dcd4fe 2024-11-24T22:16:33,628 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bd6592c37bb2:0-0 {event_type=RS_FLUSH_REGIONS, pid=155}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/8ca644d2cada6dc44b363b7383dcd4fe/.tmp/A/fceb773ae4244af2814973d66416d959, store: [table=TestAcidGuarantees family=A region=8ca644d2cada6dc44b363b7383dcd4fe] 2024-11-24T22:16:33,629 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bd6592c37bb2:0-0 {event_type=RS_FLUSH_REGIONS, pid=155}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/8ca644d2cada6dc44b363b7383dcd4fe/.tmp/A/fceb773ae4244af2814973d66416d959 is 175, key is test_row_0/A:col10/1732486591911/Put/seqid=0 2024-11-24T22:16:33,632 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45929 is added to blk_1073742403_1579 (size=30955) 2024-11-24T22:16:33,633 INFO [RS_FLUSH_OPERATIONS-regionserver/bd6592c37bb2:0-0 {event_type=RS_FLUSH_REGIONS, pid=155}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=96, memsize=40.3 K, hasBloomFilter=true, into tmp file hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/8ca644d2cada6dc44b363b7383dcd4fe/.tmp/A/fceb773ae4244af2814973d66416d959 2024-11-24T22:16:33,639 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bd6592c37bb2:0-0 {event_type=RS_FLUSH_REGIONS, pid=155}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/8ca644d2cada6dc44b363b7383dcd4fe/.tmp/B/0872fb6cf5634f7c97a9662491517a97 is 50, key is test_row_0/B:col10/1732486591911/Put/seqid=0 2024-11-24T22:16:33,642 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45929 is added to blk_1073742404_1580 (size=12001) 2024-11-24T22:16:33,741 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37701 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=154 2024-11-24T22:16:33,887 WARN [HBase-Metrics2-1 {}] impl.MetricsConfig(138): Cannot locate configuration: tried hadoop-metrics2-hbase.properties,hadoop-metrics2.properties 2024-11-24T22:16:33,910 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_default_table_TestAcidGuarantees 2024-11-24T22:16:33,910 INFO [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=TableRequests_Namespace_default_table_TestAcidGuarantees Metrics about Tables on a single HBase RegionServer 2024-11-24T22:16:34,024 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40087 {}] regionserver.HRegion(8855): Flush requested on 8ca644d2cada6dc44b363b7383dcd4fe 2024-11-24T22:16:34,024 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732486585339.8ca644d2cada6dc44b363b7383dcd4fe. as already flushing 2024-11-24T22:16:34,043 INFO [RS_FLUSH_OPERATIONS-regionserver/bd6592c37bb2:0-0 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=40.25 KB at sequenceid=96 (bloomFilter=true), to=hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/8ca644d2cada6dc44b363b7383dcd4fe/.tmp/B/0872fb6cf5634f7c97a9662491517a97 2024-11-24T22:16:34,050 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bd6592c37bb2:0-0 {event_type=RS_FLUSH_REGIONS, pid=155}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/8ca644d2cada6dc44b363b7383dcd4fe/.tmp/C/f1b82da7ecb14b54b6c5cd9a43234c81 is 50, key is test_row_0/C:col10/1732486591911/Put/seqid=0 2024-11-24T22:16:34,051 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40087 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8ca644d2cada6dc44b363b7383dcd4fe, server=bd6592c37bb2,40087,1732486423713 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-24T22:16:34,051 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40087 {}] ipc.CallRunner(138): callId: 29 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49060 deadline: 1732486654047, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8ca644d2cada6dc44b363b7383dcd4fe, server=bd6592c37bb2,40087,1732486423713 2024-11-24T22:16:34,052 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732486585339.8ca644d2cada6dc44b363b7383dcd4fe., hostname=bd6592c37bb2,40087,1732486423713, seqNum=5 , the old value is region=TestAcidGuarantees,,1732486585339.8ca644d2cada6dc44b363b7383dcd4fe., hostname=bd6592c37bb2,40087,1732486423713, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8ca644d2cada6dc44b363b7383dcd4fe, server=bd6592c37bb2,40087,1732486423713 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T22:16:34,052 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732486585339.8ca644d2cada6dc44b363b7383dcd4fe., hostname=bd6592c37bb2,40087,1732486423713, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8ca644d2cada6dc44b363b7383dcd4fe, server=bd6592c37bb2,40087,1732486423713 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T22:16:34,052 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732486585339.8ca644d2cada6dc44b363b7383dcd4fe., hostname=bd6592c37bb2,40087,1732486423713, seqNum=5 because the exception is null or not the one we care about 2024-11-24T22:16:34,053 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45929 is added to blk_1073742405_1581 (size=12001) 2024-11-24T22:16:34,056 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40087 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8ca644d2cada6dc44b363b7383dcd4fe, server=bd6592c37bb2,40087,1732486423713 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-24T22:16:34,056 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40087 {}] ipc.CallRunner(138): callId: 37 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49026 deadline: 1732486654051, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8ca644d2cada6dc44b363b7383dcd4fe, server=bd6592c37bb2,40087,1732486423713 2024-11-24T22:16:34,056 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732486585339.8ca644d2cada6dc44b363b7383dcd4fe., hostname=bd6592c37bb2,40087,1732486423713, seqNum=5 , the old value is region=TestAcidGuarantees,,1732486585339.8ca644d2cada6dc44b363b7383dcd4fe., hostname=bd6592c37bb2,40087,1732486423713, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8ca644d2cada6dc44b363b7383dcd4fe, server=bd6592c37bb2,40087,1732486423713 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T22:16:34,056 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732486585339.8ca644d2cada6dc44b363b7383dcd4fe., hostname=bd6592c37bb2,40087,1732486423713, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8ca644d2cada6dc44b363b7383dcd4fe, server=bd6592c37bb2,40087,1732486423713 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T22:16:34,056 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732486585339.8ca644d2cada6dc44b363b7383dcd4fe., hostname=bd6592c37bb2,40087,1732486423713, seqNum=5 because the exception is null or not the one we care about 2024-11-24T22:16:34,083 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40087 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8ca644d2cada6dc44b363b7383dcd4fe, server=bd6592c37bb2,40087,1732486423713 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-24T22:16:34,083 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40087 {}] ipc.CallRunner(138): callId: 40 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48986 deadline: 1732486654080, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8ca644d2cada6dc44b363b7383dcd4fe, server=bd6592c37bb2,40087,1732486423713 2024-11-24T22:16:34,083 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40087 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8ca644d2cada6dc44b363b7383dcd4fe, server=bd6592c37bb2,40087,1732486423713 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-24T22:16:34,083 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40087 {}] ipc.CallRunner(138): callId: 36 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49002 deadline: 1732486654080, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8ca644d2cada6dc44b363b7383dcd4fe, server=bd6592c37bb2,40087,1732486423713 2024-11-24T22:16:34,083 WARN [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRpcRetryingCaller(177): Call to bd6592c37bb2,40087,1732486423713 for 'test_row_0' in 8ca644d2cada6dc44b363b7383dcd4fe of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2167 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8ca644d2cada6dc44b363b7383dcd4fe, server=bd6592c37bb2,40087,1732486423713 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor44.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8ca644d2cada6dc44b363b7383dcd4fe, server=bd6592c37bb2,40087,1732486423713 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-24T22:16:34,084 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732486585339.8ca644d2cada6dc44b363b7383dcd4fe., hostname=bd6592c37bb2,40087,1732486423713, seqNum=5 , the old value is region=TestAcidGuarantees,,1732486585339.8ca644d2cada6dc44b363b7383dcd4fe., hostname=bd6592c37bb2,40087,1732486423713, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8ca644d2cada6dc44b363b7383dcd4fe, server=bd6592c37bb2,40087,1732486423713 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T22:16:34,084 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732486585339.8ca644d2cada6dc44b363b7383dcd4fe., hostname=bd6592c37bb2,40087,1732486423713, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8ca644d2cada6dc44b363b7383dcd4fe, server=bd6592c37bb2,40087,1732486423713 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T22:16:34,084 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732486585339.8ca644d2cada6dc44b363b7383dcd4fe., hostname=bd6592c37bb2,40087,1732486423713, seqNum=5 because the exception is null or not the one we care about 2024-11-24T22:16:34,083 WARN [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRpcRetryingCaller(177): Call to bd6592c37bb2,40087,1732486423713 for 'test_row_0' in 8ca644d2cada6dc44b363b7383dcd4fe of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2162 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8ca644d2cada6dc44b363b7383dcd4fe, server=bd6592c37bb2,40087,1732486423713 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor44.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8ca644d2cada6dc44b363b7383dcd4fe, server=bd6592c37bb2,40087,1732486423713 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-24T22:16:34,084 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732486585339.8ca644d2cada6dc44b363b7383dcd4fe., hostname=bd6592c37bb2,40087,1732486423713, seqNum=5 , the old value is region=TestAcidGuarantees,,1732486585339.8ca644d2cada6dc44b363b7383dcd4fe., hostname=bd6592c37bb2,40087,1732486423713, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8ca644d2cada6dc44b363b7383dcd4fe, server=bd6592c37bb2,40087,1732486423713 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T22:16:34,084 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732486585339.8ca644d2cada6dc44b363b7383dcd4fe., hostname=bd6592c37bb2,40087,1732486423713, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8ca644d2cada6dc44b363b7383dcd4fe, server=bd6592c37bb2,40087,1732486423713 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T22:16:34,084 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732486585339.8ca644d2cada6dc44b363b7383dcd4fe., hostname=bd6592c37bb2,40087,1732486423713, seqNum=5 because the exception is null or not the one we care about 2024-11-24T22:16:34,163 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40087 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8ca644d2cada6dc44b363b7383dcd4fe, server=bd6592c37bb2,40087,1732486423713 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-24T22:16:34,163 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40087 {}] ipc.CallRunner(138): callId: 30 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49060 deadline: 1732486654161, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8ca644d2cada6dc44b363b7383dcd4fe, server=bd6592c37bb2,40087,1732486423713 2024-11-24T22:16:34,164 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40087 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8ca644d2cada6dc44b363b7383dcd4fe, server=bd6592c37bb2,40087,1732486423713 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-24T22:16:34,164 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732486585339.8ca644d2cada6dc44b363b7383dcd4fe., hostname=bd6592c37bb2,40087,1732486423713, seqNum=5 , the old value is region=TestAcidGuarantees,,1732486585339.8ca644d2cada6dc44b363b7383dcd4fe., hostname=bd6592c37bb2,40087,1732486423713, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8ca644d2cada6dc44b363b7383dcd4fe, server=bd6592c37bb2,40087,1732486423713 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T22:16:34,164 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40087 {}] ipc.CallRunner(138): callId: 38 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49026 deadline: 1732486654161, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8ca644d2cada6dc44b363b7383dcd4fe, server=bd6592c37bb2,40087,1732486423713 2024-11-24T22:16:34,164 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732486585339.8ca644d2cada6dc44b363b7383dcd4fe., hostname=bd6592c37bb2,40087,1732486423713, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8ca644d2cada6dc44b363b7383dcd4fe, server=bd6592c37bb2,40087,1732486423713 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T22:16:34,164 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732486585339.8ca644d2cada6dc44b363b7383dcd4fe., hostname=bd6592c37bb2,40087,1732486423713, seqNum=5 because the exception is null or not the one we care about 2024-11-24T22:16:34,164 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732486585339.8ca644d2cada6dc44b363b7383dcd4fe., hostname=bd6592c37bb2,40087,1732486423713, seqNum=5 , the old value is region=TestAcidGuarantees,,1732486585339.8ca644d2cada6dc44b363b7383dcd4fe., hostname=bd6592c37bb2,40087,1732486423713, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8ca644d2cada6dc44b363b7383dcd4fe, server=bd6592c37bb2,40087,1732486423713 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T22:16:34,164 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732486585339.8ca644d2cada6dc44b363b7383dcd4fe., hostname=bd6592c37bb2,40087,1732486423713, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8ca644d2cada6dc44b363b7383dcd4fe, server=bd6592c37bb2,40087,1732486423713 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T22:16:34,164 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732486585339.8ca644d2cada6dc44b363b7383dcd4fe., hostname=bd6592c37bb2,40087,1732486423713, seqNum=5 because the exception is null or not the one we care about 2024-11-24T22:16:34,374 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40087 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8ca644d2cada6dc44b363b7383dcd4fe, server=bd6592c37bb2,40087,1732486423713 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-24T22:16:34,374 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40087 {}] ipc.CallRunner(138): callId: 31 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49060 deadline: 1732486654371, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8ca644d2cada6dc44b363b7383dcd4fe, server=bd6592c37bb2,40087,1732486423713 2024-11-24T22:16:34,374 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40087 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8ca644d2cada6dc44b363b7383dcd4fe, server=bd6592c37bb2,40087,1732486423713 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-24T22:16:34,374 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40087 {}] ipc.CallRunner(138): callId: 39 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49026 deadline: 1732486654371, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8ca644d2cada6dc44b363b7383dcd4fe, server=bd6592c37bb2,40087,1732486423713 2024-11-24T22:16:34,374 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732486585339.8ca644d2cada6dc44b363b7383dcd4fe., hostname=bd6592c37bb2,40087,1732486423713, seqNum=5 , the old value is region=TestAcidGuarantees,,1732486585339.8ca644d2cada6dc44b363b7383dcd4fe., hostname=bd6592c37bb2,40087,1732486423713, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8ca644d2cada6dc44b363b7383dcd4fe, server=bd6592c37bb2,40087,1732486423713 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T22:16:34,374 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732486585339.8ca644d2cada6dc44b363b7383dcd4fe., hostname=bd6592c37bb2,40087,1732486423713, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8ca644d2cada6dc44b363b7383dcd4fe, server=bd6592c37bb2,40087,1732486423713 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T22:16:34,374 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732486585339.8ca644d2cada6dc44b363b7383dcd4fe., hostname=bd6592c37bb2,40087,1732486423713, seqNum=5 because the exception is null or not the one we care about 2024-11-24T22:16:34,375 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732486585339.8ca644d2cada6dc44b363b7383dcd4fe., hostname=bd6592c37bb2,40087,1732486423713, seqNum=5 , the old value is region=TestAcidGuarantees,,1732486585339.8ca644d2cada6dc44b363b7383dcd4fe., hostname=bd6592c37bb2,40087,1732486423713, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8ca644d2cada6dc44b363b7383dcd4fe, server=bd6592c37bb2,40087,1732486423713 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T22:16:34,375 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732486585339.8ca644d2cada6dc44b363b7383dcd4fe., hostname=bd6592c37bb2,40087,1732486423713, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8ca644d2cada6dc44b363b7383dcd4fe, server=bd6592c37bb2,40087,1732486423713 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T22:16:34,375 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732486585339.8ca644d2cada6dc44b363b7383dcd4fe., hostname=bd6592c37bb2,40087,1732486423713, seqNum=5 because the exception is null or not the one we care about 2024-11-24T22:16:34,454 INFO [RS_FLUSH_OPERATIONS-regionserver/bd6592c37bb2:0-0 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=40.25 KB at sequenceid=96 (bloomFilter=true), to=hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/8ca644d2cada6dc44b363b7383dcd4fe/.tmp/C/f1b82da7ecb14b54b6c5cd9a43234c81 2024-11-24T22:16:34,457 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bd6592c37bb2:0-0 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/8ca644d2cada6dc44b363b7383dcd4fe/.tmp/A/fceb773ae4244af2814973d66416d959 as hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/8ca644d2cada6dc44b363b7383dcd4fe/A/fceb773ae4244af2814973d66416d959 2024-11-24T22:16:34,460 INFO [RS_FLUSH_OPERATIONS-regionserver/bd6592c37bb2:0-0 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/8ca644d2cada6dc44b363b7383dcd4fe/A/fceb773ae4244af2814973d66416d959, entries=150, sequenceid=96, filesize=30.2 K 2024-11-24T22:16:34,461 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bd6592c37bb2:0-0 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/8ca644d2cada6dc44b363b7383dcd4fe/.tmp/B/0872fb6cf5634f7c97a9662491517a97 as hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/8ca644d2cada6dc44b363b7383dcd4fe/B/0872fb6cf5634f7c97a9662491517a97 2024-11-24T22:16:34,464 INFO [RS_FLUSH_OPERATIONS-regionserver/bd6592c37bb2:0-0 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/8ca644d2cada6dc44b363b7383dcd4fe/B/0872fb6cf5634f7c97a9662491517a97, entries=150, sequenceid=96, filesize=11.7 K 2024-11-24T22:16:34,465 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bd6592c37bb2:0-0 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/8ca644d2cada6dc44b363b7383dcd4fe/.tmp/C/f1b82da7ecb14b54b6c5cd9a43234c81 as hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/8ca644d2cada6dc44b363b7383dcd4fe/C/f1b82da7ecb14b54b6c5cd9a43234c81 2024-11-24T22:16:34,468 INFO [RS_FLUSH_OPERATIONS-regionserver/bd6592c37bb2:0-0 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/8ca644d2cada6dc44b363b7383dcd4fe/C/f1b82da7ecb14b54b6c5cd9a43234c81, entries=150, sequenceid=96, filesize=11.7 K 2024-11-24T22:16:34,468 INFO [RS_FLUSH_OPERATIONS-regionserver/bd6592c37bb2:0-0 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.HRegion(3140): Finished flush of dataSize ~120.76 KB/123660, heapSize ~345.09 KB/353376, currentSize=60.38 KB/61830 for 8ca644d2cada6dc44b363b7383dcd4fe in 1254ms, sequenceid=96, compaction requested=false 2024-11-24T22:16:34,468 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bd6592c37bb2:0-0 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.HRegion(2603): Flush status journal for 8ca644d2cada6dc44b363b7383dcd4fe: 2024-11-24T22:16:34,468 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bd6592c37bb2:0-0 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732486585339.8ca644d2cada6dc44b363b7383dcd4fe. 2024-11-24T22:16:34,468 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bd6592c37bb2:0-0 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=155 2024-11-24T22:16:34,469 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37701 {}] master.HMaster(4169): Remote procedure done, pid=155 2024-11-24T22:16:34,471 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=155, resume processing ppid=154 2024-11-24T22:16:34,471 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=155, ppid=154, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.8670 sec 2024-11-24T22:16:34,473 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=154, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=154, table=TestAcidGuarantees in 1.8710 sec 2024-11-24T22:16:34,687 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40087 {}] regionserver.HRegion(8855): Flush requested on 8ca644d2cada6dc44b363b7383dcd4fe 2024-11-24T22:16:34,687 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 8ca644d2cada6dc44b363b7383dcd4fe 3/3 column families, dataSize=67.09 KB heapSize=199.97 KB 2024-11-24T22:16:34,687 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 8ca644d2cada6dc44b363b7383dcd4fe, store=A 2024-11-24T22:16:34,687 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-24T22:16:34,687 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 8ca644d2cada6dc44b363b7383dcd4fe, store=B 2024-11-24T22:16:34,687 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-24T22:16:34,687 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 8ca644d2cada6dc44b363b7383dcd4fe, store=C 2024-11-24T22:16:34,687 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-24T22:16:34,694 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241124055bd7d48f9849e8aebf06de5bc74b92_8ca644d2cada6dc44b363b7383dcd4fe is 50, key is test_row_0/A:col10/1732486594023/Put/seqid=0 2024-11-24T22:16:34,697 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45929 is added to blk_1073742406_1582 (size=14594) 2024-11-24T22:16:34,732 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40087 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8ca644d2cada6dc44b363b7383dcd4fe, server=bd6592c37bb2,40087,1732486423713 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-24T22:16:34,732 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40087 {}] ipc.CallRunner(138): callId: 40 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49060 deadline: 1732486654727, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8ca644d2cada6dc44b363b7383dcd4fe, server=bd6592c37bb2,40087,1732486423713 2024-11-24T22:16:34,732 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732486585339.8ca644d2cada6dc44b363b7383dcd4fe., hostname=bd6592c37bb2,40087,1732486423713, seqNum=5 , the old value is region=TestAcidGuarantees,,1732486585339.8ca644d2cada6dc44b363b7383dcd4fe., hostname=bd6592c37bb2,40087,1732486423713, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8ca644d2cada6dc44b363b7383dcd4fe, server=bd6592c37bb2,40087,1732486423713 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T22:16:34,732 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732486585339.8ca644d2cada6dc44b363b7383dcd4fe., hostname=bd6592c37bb2,40087,1732486423713, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8ca644d2cada6dc44b363b7383dcd4fe, server=bd6592c37bb2,40087,1732486423713 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T22:16:34,732 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732486585339.8ca644d2cada6dc44b363b7383dcd4fe., hostname=bd6592c37bb2,40087,1732486423713, seqNum=5 because the exception is null or not the one we care about 2024-11-24T22:16:34,733 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40087 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8ca644d2cada6dc44b363b7383dcd4fe, server=bd6592c37bb2,40087,1732486423713 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-24T22:16:34,734 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40087 {}] ipc.CallRunner(138): callId: 49 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49026 deadline: 1732486654730, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8ca644d2cada6dc44b363b7383dcd4fe, server=bd6592c37bb2,40087,1732486423713 2024-11-24T22:16:34,734 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732486585339.8ca644d2cada6dc44b363b7383dcd4fe., hostname=bd6592c37bb2,40087,1732486423713, seqNum=5 , the old value is region=TestAcidGuarantees,,1732486585339.8ca644d2cada6dc44b363b7383dcd4fe., hostname=bd6592c37bb2,40087,1732486423713, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8ca644d2cada6dc44b363b7383dcd4fe, server=bd6592c37bb2,40087,1732486423713 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T22:16:34,734 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732486585339.8ca644d2cada6dc44b363b7383dcd4fe., hostname=bd6592c37bb2,40087,1732486423713, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8ca644d2cada6dc44b363b7383dcd4fe, server=bd6592c37bb2,40087,1732486423713 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T22:16:34,734 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732486585339.8ca644d2cada6dc44b363b7383dcd4fe., hostname=bd6592c37bb2,40087,1732486423713, seqNum=5 because the exception is null or not the one we care about 2024-11-24T22:16:34,751 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37701 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=154 2024-11-24T22:16:34,751 INFO [RPCClient-NioEventLoopGroup-4-12 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-24T22:16:34,751 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37701 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-24T22:16:34,752 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37701 {}] procedure2.ProcedureExecutor(1139): Stored pid=156, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=156, table=TestAcidGuarantees 2024-11-24T22:16:34,752 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37701 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=156 2024-11-24T22:16:34,753 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=156, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=156, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-24T22:16:34,753 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=156, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=156, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-24T22:16:34,753 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=157, ppid=156, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-24T22:16:34,844 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40087 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8ca644d2cada6dc44b363b7383dcd4fe, server=bd6592c37bb2,40087,1732486423713 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-24T22:16:34,844 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40087 {}] ipc.CallRunner(138): callId: 41 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49060 deadline: 1732486654840, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8ca644d2cada6dc44b363b7383dcd4fe, server=bd6592c37bb2,40087,1732486423713 2024-11-24T22:16:34,844 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732486585339.8ca644d2cada6dc44b363b7383dcd4fe., hostname=bd6592c37bb2,40087,1732486423713, seqNum=5 , the old value is region=TestAcidGuarantees,,1732486585339.8ca644d2cada6dc44b363b7383dcd4fe., hostname=bd6592c37bb2,40087,1732486423713, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8ca644d2cada6dc44b363b7383dcd4fe, server=bd6592c37bb2,40087,1732486423713 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T22:16:34,844 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40087 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8ca644d2cada6dc44b363b7383dcd4fe, server=bd6592c37bb2,40087,1732486423713 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-24T22:16:34,844 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732486585339.8ca644d2cada6dc44b363b7383dcd4fe., hostname=bd6592c37bb2,40087,1732486423713, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8ca644d2cada6dc44b363b7383dcd4fe, server=bd6592c37bb2,40087,1732486423713 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T22:16:34,844 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732486585339.8ca644d2cada6dc44b363b7383dcd4fe., hostname=bd6592c37bb2,40087,1732486423713, seqNum=5 because the exception is null or not the one we care about 2024-11-24T22:16:34,844 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40087 {}] ipc.CallRunner(138): callId: 50 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49026 deadline: 1732486654840, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8ca644d2cada6dc44b363b7383dcd4fe, server=bd6592c37bb2,40087,1732486423713 2024-11-24T22:16:34,844 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732486585339.8ca644d2cada6dc44b363b7383dcd4fe., hostname=bd6592c37bb2,40087,1732486423713, seqNum=5 , the old value is region=TestAcidGuarantees,,1732486585339.8ca644d2cada6dc44b363b7383dcd4fe., hostname=bd6592c37bb2,40087,1732486423713, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8ca644d2cada6dc44b363b7383dcd4fe, server=bd6592c37bb2,40087,1732486423713 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T22:16:34,844 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732486585339.8ca644d2cada6dc44b363b7383dcd4fe., hostname=bd6592c37bb2,40087,1732486423713, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8ca644d2cada6dc44b363b7383dcd4fe, server=bd6592c37bb2,40087,1732486423713 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T22:16:34,844 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732486585339.8ca644d2cada6dc44b363b7383dcd4fe., hostname=bd6592c37bb2,40087,1732486423713, seqNum=5 because the exception is null or not the one we care about 2024-11-24T22:16:34,861 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37701 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=156 2024-11-24T22:16:34,905 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=40087 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=157 2024-11-24T22:16:34,905 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bd6592c37bb2:0-1 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732486585339.8ca644d2cada6dc44b363b7383dcd4fe. 2024-11-24T22:16:34,905 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bd6592c37bb2:0-1 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732486585339.8ca644d2cada6dc44b363b7383dcd4fe. as already flushing 2024-11-24T22:16:34,905 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bd6592c37bb2:0-1 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732486585339.8ca644d2cada6dc44b363b7383dcd4fe. 2024-11-24T22:16:34,905 ERROR [RS_FLUSH_OPERATIONS-regionserver/bd6592c37bb2:0-1 {event_type=RS_FLUSH_REGIONS, pid=157}] handler.RSProcedureHandler(58): pid=157 java.io.IOException: Unable to complete flush {ENCODED => 8ca644d2cada6dc44b363b7383dcd4fe, NAME => 'TestAcidGuarantees,,1732486585339.8ca644d2cada6dc44b363b7383dcd4fe.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T22:16:34,906 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bd6592c37bb2:0-1 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=157 java.io.IOException: Unable to complete flush {ENCODED => 8ca644d2cada6dc44b363b7383dcd4fe, NAME => 'TestAcidGuarantees,,1732486585339.8ca644d2cada6dc44b363b7383dcd4fe.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T22:16:34,906 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37701 {}] master.HMaster(4177): Remote procedure failed, pid=157 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 8ca644d2cada6dc44b363b7383dcd4fe, NAME => 'TestAcidGuarantees,,1732486585339.8ca644d2cada6dc44b363b7383dcd4fe.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 8ca644d2cada6dc44b363b7383dcd4fe, NAME => 'TestAcidGuarantees,,1732486585339.8ca644d2cada6dc44b363b7383dcd4fe.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T22:16:35,053 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40087 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8ca644d2cada6dc44b363b7383dcd4fe, server=bd6592c37bb2,40087,1732486423713 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-24T22:16:35,053 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40087 {}] ipc.CallRunner(138): callId: 42 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49060 deadline: 1732486655050, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8ca644d2cada6dc44b363b7383dcd4fe, server=bd6592c37bb2,40087,1732486423713 2024-11-24T22:16:35,053 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40087 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8ca644d2cada6dc44b363b7383dcd4fe, server=bd6592c37bb2,40087,1732486423713 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-24T22:16:35,054 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40087 {}] ipc.CallRunner(138): callId: 51 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49026 deadline: 1732486655050, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8ca644d2cada6dc44b363b7383dcd4fe, server=bd6592c37bb2,40087,1732486423713 2024-11-24T22:16:35,054 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732486585339.8ca644d2cada6dc44b363b7383dcd4fe., hostname=bd6592c37bb2,40087,1732486423713, seqNum=5 , the old value is region=TestAcidGuarantees,,1732486585339.8ca644d2cada6dc44b363b7383dcd4fe., hostname=bd6592c37bb2,40087,1732486423713, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8ca644d2cada6dc44b363b7383dcd4fe, server=bd6592c37bb2,40087,1732486423713 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T22:16:35,054 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732486585339.8ca644d2cada6dc44b363b7383dcd4fe., hostname=bd6592c37bb2,40087,1732486423713, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8ca644d2cada6dc44b363b7383dcd4fe, server=bd6592c37bb2,40087,1732486423713 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T22:16:35,054 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732486585339.8ca644d2cada6dc44b363b7383dcd4fe., hostname=bd6592c37bb2,40087,1732486423713, seqNum=5 because the exception is null or not the one we care about 2024-11-24T22:16:35,054 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732486585339.8ca644d2cada6dc44b363b7383dcd4fe., hostname=bd6592c37bb2,40087,1732486423713, seqNum=5 , the old value is region=TestAcidGuarantees,,1732486585339.8ca644d2cada6dc44b363b7383dcd4fe., hostname=bd6592c37bb2,40087,1732486423713, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8ca644d2cada6dc44b363b7383dcd4fe, server=bd6592c37bb2,40087,1732486423713 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T22:16:35,054 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732486585339.8ca644d2cada6dc44b363b7383dcd4fe., hostname=bd6592c37bb2,40087,1732486423713, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8ca644d2cada6dc44b363b7383dcd4fe, server=bd6592c37bb2,40087,1732486423713 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T22:16:35,054 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732486585339.8ca644d2cada6dc44b363b7383dcd4fe., hostname=bd6592c37bb2,40087,1732486423713, seqNum=5 because the exception is null or not the one we care about 2024-11-24T22:16:35,058 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=40087 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=157 2024-11-24T22:16:35,058 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bd6592c37bb2:0-2 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732486585339.8ca644d2cada6dc44b363b7383dcd4fe. 2024-11-24T22:16:35,058 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bd6592c37bb2:0-2 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732486585339.8ca644d2cada6dc44b363b7383dcd4fe. as already flushing 2024-11-24T22:16:35,058 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bd6592c37bb2:0-2 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732486585339.8ca644d2cada6dc44b363b7383dcd4fe. 2024-11-24T22:16:35,058 ERROR [RS_FLUSH_OPERATIONS-regionserver/bd6592c37bb2:0-2 {event_type=RS_FLUSH_REGIONS, pid=157}] handler.RSProcedureHandler(58): pid=157 java.io.IOException: Unable to complete flush {ENCODED => 8ca644d2cada6dc44b363b7383dcd4fe, NAME => 'TestAcidGuarantees,,1732486585339.8ca644d2cada6dc44b363b7383dcd4fe.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T22:16:35,058 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bd6592c37bb2:0-2 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=157 java.io.IOException: Unable to complete flush {ENCODED => 8ca644d2cada6dc44b363b7383dcd4fe, NAME => 'TestAcidGuarantees,,1732486585339.8ca644d2cada6dc44b363b7383dcd4fe.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T22:16:35,059 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37701 {}] master.HMaster(4177): Remote procedure failed, pid=157 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 8ca644d2cada6dc44b363b7383dcd4fe, NAME => 'TestAcidGuarantees,,1732486585339.8ca644d2cada6dc44b363b7383dcd4fe.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 8ca644d2cada6dc44b363b7383dcd4fe, NAME => 'TestAcidGuarantees,,1732486585339.8ca644d2cada6dc44b363b7383dcd4fe.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T22:16:35,071 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37701 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=156 2024-11-24T22:16:35,097 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T22:16:35,100 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241124055bd7d48f9849e8aebf06de5bc74b92_8ca644d2cada6dc44b363b7383dcd4fe to hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241124055bd7d48f9849e8aebf06de5bc74b92_8ca644d2cada6dc44b363b7383dcd4fe 2024-11-24T22:16:35,101 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/8ca644d2cada6dc44b363b7383dcd4fe/.tmp/A/c30dfe64f4b24555a71a47fb4cd35e11, store: [table=TestAcidGuarantees family=A region=8ca644d2cada6dc44b363b7383dcd4fe] 2024-11-24T22:16:35,102 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/8ca644d2cada6dc44b363b7383dcd4fe/.tmp/A/c30dfe64f4b24555a71a47fb4cd35e11 is 175, key is test_row_0/A:col10/1732486594023/Put/seqid=0 2024-11-24T22:16:35,105 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45929 is added to blk_1073742407_1583 (size=39549) 2024-11-24T22:16:35,210 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=40087 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=157 2024-11-24T22:16:35,211 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bd6592c37bb2:0-0 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732486585339.8ca644d2cada6dc44b363b7383dcd4fe. 2024-11-24T22:16:35,211 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bd6592c37bb2:0-0 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732486585339.8ca644d2cada6dc44b363b7383dcd4fe. as already flushing 2024-11-24T22:16:35,211 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bd6592c37bb2:0-0 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732486585339.8ca644d2cada6dc44b363b7383dcd4fe. 2024-11-24T22:16:35,211 ERROR [RS_FLUSH_OPERATIONS-regionserver/bd6592c37bb2:0-0 {event_type=RS_FLUSH_REGIONS, pid=157}] handler.RSProcedureHandler(58): pid=157 java.io.IOException: Unable to complete flush {ENCODED => 8ca644d2cada6dc44b363b7383dcd4fe, NAME => 'TestAcidGuarantees,,1732486585339.8ca644d2cada6dc44b363b7383dcd4fe.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T22:16:35,211 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bd6592c37bb2:0-0 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=157 java.io.IOException: Unable to complete flush {ENCODED => 8ca644d2cada6dc44b363b7383dcd4fe, NAME => 'TestAcidGuarantees,,1732486585339.8ca644d2cada6dc44b363b7383dcd4fe.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T22:16:35,211 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37701 {}] master.HMaster(4177): Remote procedure failed, pid=157 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 8ca644d2cada6dc44b363b7383dcd4fe, NAME => 'TestAcidGuarantees,,1732486585339.8ca644d2cada6dc44b363b7383dcd4fe.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 8ca644d2cada6dc44b363b7383dcd4fe, NAME => 'TestAcidGuarantees,,1732486585339.8ca644d2cada6dc44b363b7383dcd4fe.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T22:16:35,363 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=40087 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=157 2024-11-24T22:16:35,363 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40087 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8ca644d2cada6dc44b363b7383dcd4fe, server=bd6592c37bb2,40087,1732486423713 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-24T22:16:35,363 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bd6592c37bb2:0-1 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732486585339.8ca644d2cada6dc44b363b7383dcd4fe. 2024-11-24T22:16:35,363 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40087 {}] ipc.CallRunner(138): callId: 43 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49060 deadline: 1732486655361, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8ca644d2cada6dc44b363b7383dcd4fe, server=bd6592c37bb2,40087,1732486423713 2024-11-24T22:16:35,363 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bd6592c37bb2:0-1 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732486585339.8ca644d2cada6dc44b363b7383dcd4fe. as already flushing 2024-11-24T22:16:35,363 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bd6592c37bb2:0-1 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732486585339.8ca644d2cada6dc44b363b7383dcd4fe. 2024-11-24T22:16:35,364 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732486585339.8ca644d2cada6dc44b363b7383dcd4fe., hostname=bd6592c37bb2,40087,1732486423713, seqNum=5 , the old value is region=TestAcidGuarantees,,1732486585339.8ca644d2cada6dc44b363b7383dcd4fe., hostname=bd6592c37bb2,40087,1732486423713, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8ca644d2cada6dc44b363b7383dcd4fe, server=bd6592c37bb2,40087,1732486423713 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T22:16:35,363 ERROR [RS_FLUSH_OPERATIONS-regionserver/bd6592c37bb2:0-1 {event_type=RS_FLUSH_REGIONS, pid=157}] handler.RSProcedureHandler(58): pid=157 java.io.IOException: Unable to complete flush {ENCODED => 8ca644d2cada6dc44b363b7383dcd4fe, NAME => 'TestAcidGuarantees,,1732486585339.8ca644d2cada6dc44b363b7383dcd4fe.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T22:16:35,364 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40087 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8ca644d2cada6dc44b363b7383dcd4fe, server=bd6592c37bb2,40087,1732486423713 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-24T22:16:35,364 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732486585339.8ca644d2cada6dc44b363b7383dcd4fe., hostname=bd6592c37bb2,40087,1732486423713, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8ca644d2cada6dc44b363b7383dcd4fe, server=bd6592c37bb2,40087,1732486423713 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T22:16:35,364 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732486585339.8ca644d2cada6dc44b363b7383dcd4fe., hostname=bd6592c37bb2,40087,1732486423713, seqNum=5 because the exception is null or not the one we care about 2024-11-24T22:16:35,364 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bd6592c37bb2:0-1 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=157 java.io.IOException: Unable to complete flush {ENCODED => 8ca644d2cada6dc44b363b7383dcd4fe, NAME => 'TestAcidGuarantees,,1732486585339.8ca644d2cada6dc44b363b7383dcd4fe.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T22:16:35,364 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40087 {}] ipc.CallRunner(138): callId: 52 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49026 deadline: 1732486655361, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8ca644d2cada6dc44b363b7383dcd4fe, server=bd6592c37bb2,40087,1732486423713 2024-11-24T22:16:35,364 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732486585339.8ca644d2cada6dc44b363b7383dcd4fe., hostname=bd6592c37bb2,40087,1732486423713, seqNum=5 , the old value is region=TestAcidGuarantees,,1732486585339.8ca644d2cada6dc44b363b7383dcd4fe., hostname=bd6592c37bb2,40087,1732486423713, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8ca644d2cada6dc44b363b7383dcd4fe, server=bd6592c37bb2,40087,1732486423713 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T22:16:35,364 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732486585339.8ca644d2cada6dc44b363b7383dcd4fe., hostname=bd6592c37bb2,40087,1732486423713, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8ca644d2cada6dc44b363b7383dcd4fe, server=bd6592c37bb2,40087,1732486423713 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T22:16:35,364 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732486585339.8ca644d2cada6dc44b363b7383dcd4fe., hostname=bd6592c37bb2,40087,1732486423713, seqNum=5 because the exception is null or not the one we care about 2024-11-24T22:16:35,364 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37701 {}] master.HMaster(4177): Remote procedure failed, pid=157 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 8ca644d2cada6dc44b363b7383dcd4fe, NAME => 'TestAcidGuarantees,,1732486585339.8ca644d2cada6dc44b363b7383dcd4fe.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 8ca644d2cada6dc44b363b7383dcd4fe, NAME => 'TestAcidGuarantees,,1732486585339.8ca644d2cada6dc44b363b7383dcd4fe.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T22:16:35,381 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37701 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=156 2024-11-24T22:16:35,506 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=113, memsize=24.6 K, hasBloomFilter=true, into tmp file hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/8ca644d2cada6dc44b363b7383dcd4fe/.tmp/A/c30dfe64f4b24555a71a47fb4cd35e11 2024-11-24T22:16:35,512 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/8ca644d2cada6dc44b363b7383dcd4fe/.tmp/B/38483da317664eb19a79deb055a938d4 is 50, key is test_row_0/B:col10/1732486594023/Put/seqid=0 2024-11-24T22:16:35,516 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45929 is added to blk_1073742408_1584 (size=12001) 2024-11-24T22:16:35,517 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=40087 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=157 2024-11-24T22:16:35,517 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bd6592c37bb2:0-2 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732486585339.8ca644d2cada6dc44b363b7383dcd4fe. 2024-11-24T22:16:35,517 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bd6592c37bb2:0-2 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732486585339.8ca644d2cada6dc44b363b7383dcd4fe. as already flushing 2024-11-24T22:16:35,517 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bd6592c37bb2:0-2 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732486585339.8ca644d2cada6dc44b363b7383dcd4fe. 2024-11-24T22:16:35,517 ERROR [RS_FLUSH_OPERATIONS-regionserver/bd6592c37bb2:0-2 {event_type=RS_FLUSH_REGIONS, pid=157}] handler.RSProcedureHandler(58): pid=157 java.io.IOException: Unable to complete flush {ENCODED => 8ca644d2cada6dc44b363b7383dcd4fe, NAME => 'TestAcidGuarantees,,1732486585339.8ca644d2cada6dc44b363b7383dcd4fe.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T22:16:35,517 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bd6592c37bb2:0-2 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=157 java.io.IOException: Unable to complete flush {ENCODED => 8ca644d2cada6dc44b363b7383dcd4fe, NAME => 'TestAcidGuarantees,,1732486585339.8ca644d2cada6dc44b363b7383dcd4fe.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T22:16:35,518 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37701 {}] master.HMaster(4177): Remote procedure failed, pid=157 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 8ca644d2cada6dc44b363b7383dcd4fe, NAME => 'TestAcidGuarantees,,1732486585339.8ca644d2cada6dc44b363b7383dcd4fe.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 8ca644d2cada6dc44b363b7383dcd4fe, NAME => 'TestAcidGuarantees,,1732486585339.8ca644d2cada6dc44b363b7383dcd4fe.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T22:16:35,670 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=40087 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=157 2024-11-24T22:16:35,670 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bd6592c37bb2:0-0 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732486585339.8ca644d2cada6dc44b363b7383dcd4fe. 2024-11-24T22:16:35,670 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bd6592c37bb2:0-0 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732486585339.8ca644d2cada6dc44b363b7383dcd4fe. as already flushing 2024-11-24T22:16:35,670 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bd6592c37bb2:0-0 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732486585339.8ca644d2cada6dc44b363b7383dcd4fe. 2024-11-24T22:16:35,670 ERROR [RS_FLUSH_OPERATIONS-regionserver/bd6592c37bb2:0-0 {event_type=RS_FLUSH_REGIONS, pid=157}] handler.RSProcedureHandler(58): pid=157 java.io.IOException: Unable to complete flush {ENCODED => 8ca644d2cada6dc44b363b7383dcd4fe, NAME => 'TestAcidGuarantees,,1732486585339.8ca644d2cada6dc44b363b7383dcd4fe.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T22:16:35,670 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bd6592c37bb2:0-0 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=157 java.io.IOException: Unable to complete flush {ENCODED => 8ca644d2cada6dc44b363b7383dcd4fe, NAME => 'TestAcidGuarantees,,1732486585339.8ca644d2cada6dc44b363b7383dcd4fe.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T22:16:35,671 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37701 {}] master.HMaster(4177): Remote procedure failed, pid=157 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 8ca644d2cada6dc44b363b7383dcd4fe, NAME => 'TestAcidGuarantees,,1732486585339.8ca644d2cada6dc44b363b7383dcd4fe.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 8ca644d2cada6dc44b363b7383dcd4fe, NAME => 'TestAcidGuarantees,,1732486585339.8ca644d2cada6dc44b363b7383dcd4fe.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T22:16:35,821 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=40087 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=157 2024-11-24T22:16:35,821 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bd6592c37bb2:0-1 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732486585339.8ca644d2cada6dc44b363b7383dcd4fe. 2024-11-24T22:16:35,821 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bd6592c37bb2:0-1 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732486585339.8ca644d2cada6dc44b363b7383dcd4fe. as already flushing 2024-11-24T22:16:35,821 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bd6592c37bb2:0-1 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732486585339.8ca644d2cada6dc44b363b7383dcd4fe. 2024-11-24T22:16:35,822 ERROR [RS_FLUSH_OPERATIONS-regionserver/bd6592c37bb2:0-1 {event_type=RS_FLUSH_REGIONS, pid=157}] handler.RSProcedureHandler(58): pid=157 java.io.IOException: Unable to complete flush {ENCODED => 8ca644d2cada6dc44b363b7383dcd4fe, NAME => 'TestAcidGuarantees,,1732486585339.8ca644d2cada6dc44b363b7383dcd4fe.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T22:16:35,822 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bd6592c37bb2:0-1 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=157 java.io.IOException: Unable to complete flush {ENCODED => 8ca644d2cada6dc44b363b7383dcd4fe, NAME => 'TestAcidGuarantees,,1732486585339.8ca644d2cada6dc44b363b7383dcd4fe.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T22:16:35,822 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37701 {}] master.HMaster(4177): Remote procedure failed, pid=157 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 8ca644d2cada6dc44b363b7383dcd4fe, NAME => 'TestAcidGuarantees,,1732486585339.8ca644d2cada6dc44b363b7383dcd4fe.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 8ca644d2cada6dc44b363b7383dcd4fe, NAME => 'TestAcidGuarantees,,1732486585339.8ca644d2cada6dc44b363b7383dcd4fe.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T22:16:35,874 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40087 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8ca644d2cada6dc44b363b7383dcd4fe, server=bd6592c37bb2,40087,1732486423713 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-24T22:16:35,874 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40087 {}] ipc.CallRunner(138): callId: 44 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49060 deadline: 1732486655870, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8ca644d2cada6dc44b363b7383dcd4fe, server=bd6592c37bb2,40087,1732486423713 2024-11-24T22:16:35,875 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40087 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8ca644d2cada6dc44b363b7383dcd4fe, server=bd6592c37bb2,40087,1732486423713 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-24T22:16:35,875 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40087 {}] ipc.CallRunner(138): callId: 53 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49026 deadline: 1732486655871, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8ca644d2cada6dc44b363b7383dcd4fe, server=bd6592c37bb2,40087,1732486423713 2024-11-24T22:16:35,875 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732486585339.8ca644d2cada6dc44b363b7383dcd4fe., hostname=bd6592c37bb2,40087,1732486423713, seqNum=5 , the old value is region=TestAcidGuarantees,,1732486585339.8ca644d2cada6dc44b363b7383dcd4fe., hostname=bd6592c37bb2,40087,1732486423713, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8ca644d2cada6dc44b363b7383dcd4fe, server=bd6592c37bb2,40087,1732486423713 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T22:16:35,875 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732486585339.8ca644d2cada6dc44b363b7383dcd4fe., hostname=bd6592c37bb2,40087,1732486423713, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8ca644d2cada6dc44b363b7383dcd4fe, server=bd6592c37bb2,40087,1732486423713 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T22:16:35,875 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732486585339.8ca644d2cada6dc44b363b7383dcd4fe., hostname=bd6592c37bb2,40087,1732486423713, seqNum=5 because the exception is null or not the one we care about 2024-11-24T22:16:35,875 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732486585339.8ca644d2cada6dc44b363b7383dcd4fe., hostname=bd6592c37bb2,40087,1732486423713, seqNum=5 , the old value is region=TestAcidGuarantees,,1732486585339.8ca644d2cada6dc44b363b7383dcd4fe., hostname=bd6592c37bb2,40087,1732486423713, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8ca644d2cada6dc44b363b7383dcd4fe, server=bd6592c37bb2,40087,1732486423713 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T22:16:35,875 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732486585339.8ca644d2cada6dc44b363b7383dcd4fe., hostname=bd6592c37bb2,40087,1732486423713, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8ca644d2cada6dc44b363b7383dcd4fe, server=bd6592c37bb2,40087,1732486423713 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T22:16:35,875 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732486585339.8ca644d2cada6dc44b363b7383dcd4fe., hostname=bd6592c37bb2,40087,1732486423713, seqNum=5 because the exception is null or not the one we care about 2024-11-24T22:16:35,891 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37701 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=156 2024-11-24T22:16:35,917 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=24.60 KB at sequenceid=113 (bloomFilter=true), to=hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/8ca644d2cada6dc44b363b7383dcd4fe/.tmp/B/38483da317664eb19a79deb055a938d4 2024-11-24T22:16:35,924 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/8ca644d2cada6dc44b363b7383dcd4fe/.tmp/C/fbb52ae7a00f44df87fcf504039c6b4a is 50, key is test_row_0/C:col10/1732486594023/Put/seqid=0 2024-11-24T22:16:35,927 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45929 is added to blk_1073742409_1585 (size=12001) 2024-11-24T22:16:35,974 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=40087 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=157 2024-11-24T22:16:35,974 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bd6592c37bb2:0-2 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732486585339.8ca644d2cada6dc44b363b7383dcd4fe. 2024-11-24T22:16:35,974 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bd6592c37bb2:0-2 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732486585339.8ca644d2cada6dc44b363b7383dcd4fe. as already flushing 2024-11-24T22:16:35,974 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bd6592c37bb2:0-2 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732486585339.8ca644d2cada6dc44b363b7383dcd4fe. 2024-11-24T22:16:35,974 ERROR [RS_FLUSH_OPERATIONS-regionserver/bd6592c37bb2:0-2 {event_type=RS_FLUSH_REGIONS, pid=157}] handler.RSProcedureHandler(58): pid=157 java.io.IOException: Unable to complete flush {ENCODED => 8ca644d2cada6dc44b363b7383dcd4fe, NAME => 'TestAcidGuarantees,,1732486585339.8ca644d2cada6dc44b363b7383dcd4fe.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T22:16:35,974 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bd6592c37bb2:0-2 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=157 java.io.IOException: Unable to complete flush {ENCODED => 8ca644d2cada6dc44b363b7383dcd4fe, NAME => 'TestAcidGuarantees,,1732486585339.8ca644d2cada6dc44b363b7383dcd4fe.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T22:16:35,975 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37701 {}] master.HMaster(4177): Remote procedure failed, pid=157 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 8ca644d2cada6dc44b363b7383dcd4fe, NAME => 'TestAcidGuarantees,,1732486585339.8ca644d2cada6dc44b363b7383dcd4fe.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 8ca644d2cada6dc44b363b7383dcd4fe, NAME => 'TestAcidGuarantees,,1732486585339.8ca644d2cada6dc44b363b7383dcd4fe.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T22:16:36,095 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40087 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8ca644d2cada6dc44b363b7383dcd4fe, server=bd6592c37bb2,40087,1732486423713 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-24T22:16:36,095 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40087 {}] ipc.CallRunner(138): callId: 37 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49002 deadline: 1732486656090, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8ca644d2cada6dc44b363b7383dcd4fe, server=bd6592c37bb2,40087,1732486423713 2024-11-24T22:16:36,095 WARN [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRpcRetryingCaller(177): Call to bd6592c37bb2,40087,1732486423713 for 'test_row_0' in 8ca644d2cada6dc44b363b7383dcd4fe of TestAcidGuarantees failed, tries = 7, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 4174 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8ca644d2cada6dc44b363b7383dcd4fe, server=bd6592c37bb2,40087,1732486423713 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor44.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8ca644d2cada6dc44b363b7383dcd4fe, server=bd6592c37bb2,40087,1732486423713 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-24T22:16:36,096 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732486585339.8ca644d2cada6dc44b363b7383dcd4fe., hostname=bd6592c37bb2,40087,1732486423713, seqNum=5 , the old value is region=TestAcidGuarantees,,1732486585339.8ca644d2cada6dc44b363b7383dcd4fe., hostname=bd6592c37bb2,40087,1732486423713, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8ca644d2cada6dc44b363b7383dcd4fe, server=bd6592c37bb2,40087,1732486423713 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T22:16:36,096 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732486585339.8ca644d2cada6dc44b363b7383dcd4fe., hostname=bd6592c37bb2,40087,1732486423713, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8ca644d2cada6dc44b363b7383dcd4fe, server=bd6592c37bb2,40087,1732486423713 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T22:16:36,096 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732486585339.8ca644d2cada6dc44b363b7383dcd4fe., hostname=bd6592c37bb2,40087,1732486423713, seqNum=5 because the exception is null or not the one we care about 2024-11-24T22:16:36,105 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40087 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8ca644d2cada6dc44b363b7383dcd4fe, server=bd6592c37bb2,40087,1732486423713 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-24T22:16:36,105 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40087 {}] ipc.CallRunner(138): callId: 41 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48986 deadline: 1732486656101, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8ca644d2cada6dc44b363b7383dcd4fe, server=bd6592c37bb2,40087,1732486423713 2024-11-24T22:16:36,106 WARN [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRpcRetryingCaller(177): Call to bd6592c37bb2,40087,1732486423713 for 'test_row_0' in 8ca644d2cada6dc44b363b7383dcd4fe of TestAcidGuarantees failed, tries = 7, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 4189 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8ca644d2cada6dc44b363b7383dcd4fe, server=bd6592c37bb2,40087,1732486423713 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor44.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8ca644d2cada6dc44b363b7383dcd4fe, server=bd6592c37bb2,40087,1732486423713 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-24T22:16:36,106 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732486585339.8ca644d2cada6dc44b363b7383dcd4fe., hostname=bd6592c37bb2,40087,1732486423713, seqNum=5 , the old value is region=TestAcidGuarantees,,1732486585339.8ca644d2cada6dc44b363b7383dcd4fe., hostname=bd6592c37bb2,40087,1732486423713, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8ca644d2cada6dc44b363b7383dcd4fe, server=bd6592c37bb2,40087,1732486423713 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T22:16:36,106 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732486585339.8ca644d2cada6dc44b363b7383dcd4fe., hostname=bd6592c37bb2,40087,1732486423713, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8ca644d2cada6dc44b363b7383dcd4fe, server=bd6592c37bb2,40087,1732486423713 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T22:16:36,106 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732486585339.8ca644d2cada6dc44b363b7383dcd4fe., hostname=bd6592c37bb2,40087,1732486423713, seqNum=5 because the exception is null or not the one we care about 2024-11-24T22:16:36,126 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=40087 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=157 2024-11-24T22:16:36,126 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bd6592c37bb2:0-0 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732486585339.8ca644d2cada6dc44b363b7383dcd4fe. 2024-11-24T22:16:36,127 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bd6592c37bb2:0-0 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732486585339.8ca644d2cada6dc44b363b7383dcd4fe. as already flushing 2024-11-24T22:16:36,127 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bd6592c37bb2:0-0 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732486585339.8ca644d2cada6dc44b363b7383dcd4fe. 2024-11-24T22:16:36,127 ERROR [RS_FLUSH_OPERATIONS-regionserver/bd6592c37bb2:0-0 {event_type=RS_FLUSH_REGIONS, pid=157}] handler.RSProcedureHandler(58): pid=157 java.io.IOException: Unable to complete flush {ENCODED => 8ca644d2cada6dc44b363b7383dcd4fe, NAME => 'TestAcidGuarantees,,1732486585339.8ca644d2cada6dc44b363b7383dcd4fe.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T22:16:36,127 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bd6592c37bb2:0-0 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=157 java.io.IOException: Unable to complete flush {ENCODED => 8ca644d2cada6dc44b363b7383dcd4fe, NAME => 'TestAcidGuarantees,,1732486585339.8ca644d2cada6dc44b363b7383dcd4fe.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T22:16:36,127 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37701 {}] master.HMaster(4177): Remote procedure failed, pid=157 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 8ca644d2cada6dc44b363b7383dcd4fe, NAME => 'TestAcidGuarantees,,1732486585339.8ca644d2cada6dc44b363b7383dcd4fe.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 8ca644d2cada6dc44b363b7383dcd4fe, NAME => 'TestAcidGuarantees,,1732486585339.8ca644d2cada6dc44b363b7383dcd4fe.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T22:16:36,279 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=40087 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=157 2024-11-24T22:16:36,280 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bd6592c37bb2:0-1 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732486585339.8ca644d2cada6dc44b363b7383dcd4fe. 2024-11-24T22:16:36,280 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bd6592c37bb2:0-1 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732486585339.8ca644d2cada6dc44b363b7383dcd4fe. as already flushing 2024-11-24T22:16:36,280 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bd6592c37bb2:0-1 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732486585339.8ca644d2cada6dc44b363b7383dcd4fe. 2024-11-24T22:16:36,280 ERROR [RS_FLUSH_OPERATIONS-regionserver/bd6592c37bb2:0-1 {event_type=RS_FLUSH_REGIONS, pid=157}] handler.RSProcedureHandler(58): pid=157 java.io.IOException: Unable to complete flush {ENCODED => 8ca644d2cada6dc44b363b7383dcd4fe, NAME => 'TestAcidGuarantees,,1732486585339.8ca644d2cada6dc44b363b7383dcd4fe.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T22:16:36,280 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bd6592c37bb2:0-1 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=157 java.io.IOException: Unable to complete flush {ENCODED => 8ca644d2cada6dc44b363b7383dcd4fe, NAME => 'TestAcidGuarantees,,1732486585339.8ca644d2cada6dc44b363b7383dcd4fe.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T22:16:36,280 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37701 {}] master.HMaster(4177): Remote procedure failed, pid=157 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 8ca644d2cada6dc44b363b7383dcd4fe, NAME => 'TestAcidGuarantees,,1732486585339.8ca644d2cada6dc44b363b7383dcd4fe.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 8ca644d2cada6dc44b363b7383dcd4fe, NAME => 'TestAcidGuarantees,,1732486585339.8ca644d2cada6dc44b363b7383dcd4fe.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T22:16:36,330 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=24.60 KB at sequenceid=113 (bloomFilter=true), to=hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/8ca644d2cada6dc44b363b7383dcd4fe/.tmp/C/fbb52ae7a00f44df87fcf504039c6b4a 2024-11-24T22:16:36,340 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/8ca644d2cada6dc44b363b7383dcd4fe/.tmp/A/c30dfe64f4b24555a71a47fb4cd35e11 as hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/8ca644d2cada6dc44b363b7383dcd4fe/A/c30dfe64f4b24555a71a47fb4cd35e11 2024-11-24T22:16:36,344 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/8ca644d2cada6dc44b363b7383dcd4fe/A/c30dfe64f4b24555a71a47fb4cd35e11, entries=200, sequenceid=113, filesize=38.6 K 2024-11-24T22:16:36,345 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/8ca644d2cada6dc44b363b7383dcd4fe/.tmp/B/38483da317664eb19a79deb055a938d4 as hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/8ca644d2cada6dc44b363b7383dcd4fe/B/38483da317664eb19a79deb055a938d4 2024-11-24T22:16:36,347 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/8ca644d2cada6dc44b363b7383dcd4fe/B/38483da317664eb19a79deb055a938d4, entries=150, sequenceid=113, filesize=11.7 K 2024-11-24T22:16:36,348 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/8ca644d2cada6dc44b363b7383dcd4fe/.tmp/C/fbb52ae7a00f44df87fcf504039c6b4a as hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/8ca644d2cada6dc44b363b7383dcd4fe/C/fbb52ae7a00f44df87fcf504039c6b4a 2024-11-24T22:16:36,351 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/8ca644d2cada6dc44b363b7383dcd4fe/C/fbb52ae7a00f44df87fcf504039c6b4a, entries=150, sequenceid=113, filesize=11.7 K 2024-11-24T22:16:36,351 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~73.80 KB/75570, heapSize ~219.84 KB/225120, currentSize=100.63 KB/103050 for 8ca644d2cada6dc44b363b7383dcd4fe in 1664ms, sequenceid=113, compaction requested=true 2024-11-24T22:16:36,351 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 8ca644d2cada6dc44b363b7383dcd4fe: 2024-11-24T22:16:36,352 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 8ca644d2cada6dc44b363b7383dcd4fe:A, priority=-2147483648, current under compaction store size is 1 2024-11-24T22:16:36,352 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-24T22:16:36,352 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 8ca644d2cada6dc44b363b7383dcd4fe:B, priority=-2147483648, current under compaction store size is 2 2024-11-24T22:16:36,352 DEBUG [RS:0;bd6592c37bb2:40087-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-24T22:16:36,352 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-24T22:16:36,352 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 8ca644d2cada6dc44b363b7383dcd4fe:C, priority=-2147483648, current under compaction store size is 3 2024-11-24T22:16:36,352 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-24T22:16:36,352 DEBUG [RS:0;bd6592c37bb2:40087-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-24T22:16:36,352 DEBUG [RS:0;bd6592c37bb2:40087-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 101669 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-24T22:16:36,353 DEBUG [RS:0;bd6592c37bb2:40087-shortCompactions-0 {}] regionserver.HStore(1541): 8ca644d2cada6dc44b363b7383dcd4fe/A is initiating minor compaction (all files) 2024-11-24T22:16:36,353 INFO [RS:0;bd6592c37bb2:40087-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 8ca644d2cada6dc44b363b7383dcd4fe/A in TestAcidGuarantees,,1732486585339.8ca644d2cada6dc44b363b7383dcd4fe. 2024-11-24T22:16:36,353 INFO [RS:0;bd6592c37bb2:40087-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/8ca644d2cada6dc44b363b7383dcd4fe/A/ff60faeb807c4decab1b2a1f7ba31d85, hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/8ca644d2cada6dc44b363b7383dcd4fe/A/fceb773ae4244af2814973d66416d959, hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/8ca644d2cada6dc44b363b7383dcd4fe/A/c30dfe64f4b24555a71a47fb4cd35e11] into tmpdir=hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/8ca644d2cada6dc44b363b7383dcd4fe/.tmp, totalSize=99.3 K 2024-11-24T22:16:36,353 INFO [RS:0;bd6592c37bb2:40087-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=13 throughput controller=DefaultCompactionThroughputController [maxThroughput=57.69 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1732486585339.8ca644d2cada6dc44b363b7383dcd4fe. 2024-11-24T22:16:36,353 DEBUG [RS:0;bd6592c37bb2:40087-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1732486585339.8ca644d2cada6dc44b363b7383dcd4fe. files: [hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/8ca644d2cada6dc44b363b7383dcd4fe/A/ff60faeb807c4decab1b2a1f7ba31d85, hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/8ca644d2cada6dc44b363b7383dcd4fe/A/fceb773ae4244af2814973d66416d959, hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/8ca644d2cada6dc44b363b7383dcd4fe/A/c30dfe64f4b24555a71a47fb4cd35e11] 2024-11-24T22:16:36,353 DEBUG [RS:0;bd6592c37bb2:40087-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36106 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-24T22:16:36,353 DEBUG [RS:0;bd6592c37bb2:40087-longCompactions-0 {}] regionserver.HStore(1541): 8ca644d2cada6dc44b363b7383dcd4fe/B is initiating minor compaction (all files) 2024-11-24T22:16:36,353 DEBUG [RS:0;bd6592c37bb2:40087-shortCompactions-0 {}] compactions.Compactor(225): Compacting ff60faeb807c4decab1b2a1f7ba31d85, keycount=150, bloomtype=ROW, size=30.4 K, encoding=NONE, compression=NONE, seqNum=61, earliestPutTs=1732486588524 2024-11-24T22:16:36,353 INFO [RS:0;bd6592c37bb2:40087-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 8ca644d2cada6dc44b363b7383dcd4fe/B in TestAcidGuarantees,,1732486585339.8ca644d2cada6dc44b363b7383dcd4fe. 2024-11-24T22:16:36,353 INFO [RS:0;bd6592c37bb2:40087-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/8ca644d2cada6dc44b363b7383dcd4fe/B/9e0fd71ad3754b308cad5b3311354f00, hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/8ca644d2cada6dc44b363b7383dcd4fe/B/0872fb6cf5634f7c97a9662491517a97, hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/8ca644d2cada6dc44b363b7383dcd4fe/B/38483da317664eb19a79deb055a938d4] into tmpdir=hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/8ca644d2cada6dc44b363b7383dcd4fe/.tmp, totalSize=35.3 K 2024-11-24T22:16:36,354 DEBUG [RS:0;bd6592c37bb2:40087-shortCompactions-0 {}] compactions.Compactor(225): Compacting fceb773ae4244af2814973d66416d959, keycount=150, bloomtype=ROW, size=30.2 K, encoding=NONE, compression=NONE, seqNum=96, earliestPutTs=1732486591911 2024-11-24T22:16:36,354 DEBUG [RS:0;bd6592c37bb2:40087-longCompactions-0 {}] compactions.Compactor(225): Compacting 9e0fd71ad3754b308cad5b3311354f00, keycount=150, bloomtype=ROW, size=11.8 K, encoding=NONE, compression=NONE, seqNum=61, earliestPutTs=1732486588524 2024-11-24T22:16:36,354 DEBUG [RS:0;bd6592c37bb2:40087-shortCompactions-0 {}] compactions.Compactor(225): Compacting c30dfe64f4b24555a71a47fb4cd35e11, keycount=200, bloomtype=ROW, size=38.6 K, encoding=NONE, compression=NONE, seqNum=113, earliestPutTs=1732486594023 2024-11-24T22:16:36,354 DEBUG [RS:0;bd6592c37bb2:40087-longCompactions-0 {}] compactions.Compactor(225): Compacting 0872fb6cf5634f7c97a9662491517a97, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=96, earliestPutTs=1732486591911 2024-11-24T22:16:36,354 DEBUG [RS:0;bd6592c37bb2:40087-longCompactions-0 {}] compactions.Compactor(225): Compacting 38483da317664eb19a79deb055a938d4, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=113, earliestPutTs=1732486594023 2024-11-24T22:16:36,358 INFO [RS:0;bd6592c37bb2:40087-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=8ca644d2cada6dc44b363b7383dcd4fe] 2024-11-24T22:16:36,360 INFO [RS:0;bd6592c37bb2:40087-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 8ca644d2cada6dc44b363b7383dcd4fe#B#compaction#498 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 57.69 MB/second 2024-11-24T22:16:36,360 DEBUG [RS:0;bd6592c37bb2:40087-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/8ca644d2cada6dc44b363b7383dcd4fe/.tmp/B/38029836f3024bea8777a5b68b4ad875 is 50, key is test_row_0/B:col10/1732486594023/Put/seqid=0 2024-11-24T22:16:36,361 DEBUG [RS:0;bd6592c37bb2:40087-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e20241124bb85d3f7705545af898a0075d3fc3f11_8ca644d2cada6dc44b363b7383dcd4fe store=[table=TestAcidGuarantees family=A region=8ca644d2cada6dc44b363b7383dcd4fe] 2024-11-24T22:16:36,362 DEBUG [RS:0;bd6592c37bb2:40087-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e20241124bb85d3f7705545af898a0075d3fc3f11_8ca644d2cada6dc44b363b7383dcd4fe, store=[table=TestAcidGuarantees family=A region=8ca644d2cada6dc44b363b7383dcd4fe] 2024-11-24T22:16:36,363 DEBUG [RS:0;bd6592c37bb2:40087-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241124bb85d3f7705545af898a0075d3fc3f11_8ca644d2cada6dc44b363b7383dcd4fe because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=8ca644d2cada6dc44b363b7383dcd4fe] 2024-11-24T22:16:36,363 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45929 is added to blk_1073742410_1586 (size=12207) 2024-11-24T22:16:36,366 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45929 is added to blk_1073742411_1587 (size=4469) 2024-11-24T22:16:36,367 INFO [RS:0;bd6592c37bb2:40087-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 8ca644d2cada6dc44b363b7383dcd4fe#A#compaction#497 average throughput is 2.71 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 57.69 MB/second 2024-11-24T22:16:36,367 DEBUG [RS:0;bd6592c37bb2:40087-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/8ca644d2cada6dc44b363b7383dcd4fe/.tmp/A/3cadf84e7037408481c122039cbf1085 is 175, key is test_row_0/A:col10/1732486594023/Put/seqid=0 2024-11-24T22:16:36,368 DEBUG [RS:0;bd6592c37bb2:40087-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/8ca644d2cada6dc44b363b7383dcd4fe/.tmp/B/38029836f3024bea8777a5b68b4ad875 as hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/8ca644d2cada6dc44b363b7383dcd4fe/B/38029836f3024bea8777a5b68b4ad875 2024-11-24T22:16:36,372 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45929 is added to blk_1073742412_1588 (size=31161) 2024-11-24T22:16:36,374 INFO [RS:0;bd6592c37bb2:40087-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 8ca644d2cada6dc44b363b7383dcd4fe/B of 8ca644d2cada6dc44b363b7383dcd4fe into 38029836f3024bea8777a5b68b4ad875(size=11.9 K), total size for store is 11.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-24T22:16:36,374 DEBUG [RS:0;bd6592c37bb2:40087-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 8ca644d2cada6dc44b363b7383dcd4fe: 2024-11-24T22:16:36,374 INFO [RS:0;bd6592c37bb2:40087-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732486585339.8ca644d2cada6dc44b363b7383dcd4fe., storeName=8ca644d2cada6dc44b363b7383dcd4fe/B, priority=13, startTime=1732486596352; duration=0sec 2024-11-24T22:16:36,374 DEBUG [RS:0;bd6592c37bb2:40087-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-24T22:16:36,374 DEBUG [RS:0;bd6592c37bb2:40087-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 8ca644d2cada6dc44b363b7383dcd4fe:B 2024-11-24T22:16:36,374 DEBUG [RS:0;bd6592c37bb2:40087-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-24T22:16:36,375 DEBUG [RS:0;bd6592c37bb2:40087-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36106 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-24T22:16:36,375 DEBUG [RS:0;bd6592c37bb2:40087-longCompactions-0 {}] regionserver.HStore(1541): 8ca644d2cada6dc44b363b7383dcd4fe/C is initiating minor compaction (all files) 2024-11-24T22:16:36,375 INFO [RS:0;bd6592c37bb2:40087-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 8ca644d2cada6dc44b363b7383dcd4fe/C in TestAcidGuarantees,,1732486585339.8ca644d2cada6dc44b363b7383dcd4fe. 2024-11-24T22:16:36,375 INFO [RS:0;bd6592c37bb2:40087-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/8ca644d2cada6dc44b363b7383dcd4fe/C/77cfe1f3e45a4ae4a303695fd6b77f4a, hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/8ca644d2cada6dc44b363b7383dcd4fe/C/f1b82da7ecb14b54b6c5cd9a43234c81, hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/8ca644d2cada6dc44b363b7383dcd4fe/C/fbb52ae7a00f44df87fcf504039c6b4a] into tmpdir=hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/8ca644d2cada6dc44b363b7383dcd4fe/.tmp, totalSize=35.3 K 2024-11-24T22:16:36,376 DEBUG [RS:0;bd6592c37bb2:40087-longCompactions-0 {}] compactions.Compactor(225): Compacting 77cfe1f3e45a4ae4a303695fd6b77f4a, keycount=150, bloomtype=ROW, size=11.8 K, encoding=NONE, compression=NONE, seqNum=61, earliestPutTs=1732486588524 2024-11-24T22:16:36,377 DEBUG [RS:0;bd6592c37bb2:40087-longCompactions-0 {}] compactions.Compactor(225): Compacting f1b82da7ecb14b54b6c5cd9a43234c81, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=96, earliestPutTs=1732486591911 2024-11-24T22:16:36,377 DEBUG [RS:0;bd6592c37bb2:40087-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/8ca644d2cada6dc44b363b7383dcd4fe/.tmp/A/3cadf84e7037408481c122039cbf1085 as hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/8ca644d2cada6dc44b363b7383dcd4fe/A/3cadf84e7037408481c122039cbf1085 2024-11-24T22:16:36,377 DEBUG [RS:0;bd6592c37bb2:40087-longCompactions-0 {}] compactions.Compactor(225): Compacting fbb52ae7a00f44df87fcf504039c6b4a, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=113, earliestPutTs=1732486594023 2024-11-24T22:16:36,381 INFO [RS:0;bd6592c37bb2:40087-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 8ca644d2cada6dc44b363b7383dcd4fe/A of 8ca644d2cada6dc44b363b7383dcd4fe into 3cadf84e7037408481c122039cbf1085(size=30.4 K), total size for store is 30.4 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-24T22:16:36,381 DEBUG [RS:0;bd6592c37bb2:40087-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 8ca644d2cada6dc44b363b7383dcd4fe: 2024-11-24T22:16:36,381 INFO [RS:0;bd6592c37bb2:40087-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732486585339.8ca644d2cada6dc44b363b7383dcd4fe., storeName=8ca644d2cada6dc44b363b7383dcd4fe/A, priority=13, startTime=1732486596351; duration=0sec 2024-11-24T22:16:36,381 DEBUG [RS:0;bd6592c37bb2:40087-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-24T22:16:36,381 DEBUG [RS:0;bd6592c37bb2:40087-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 8ca644d2cada6dc44b363b7383dcd4fe:A 2024-11-24T22:16:36,384 INFO [RS:0;bd6592c37bb2:40087-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 8ca644d2cada6dc44b363b7383dcd4fe#C#compaction#499 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 57.69 MB/second 2024-11-24T22:16:36,384 DEBUG [RS:0;bd6592c37bb2:40087-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/8ca644d2cada6dc44b363b7383dcd4fe/.tmp/C/207460af5f1c459cbc35febbfcc81050 is 50, key is test_row_0/C:col10/1732486594023/Put/seqid=0 2024-11-24T22:16:36,387 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45929 is added to blk_1073742413_1589 (size=12207) 2024-11-24T22:16:36,391 DEBUG [RS:0;bd6592c37bb2:40087-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/8ca644d2cada6dc44b363b7383dcd4fe/.tmp/C/207460af5f1c459cbc35febbfcc81050 as hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/8ca644d2cada6dc44b363b7383dcd4fe/C/207460af5f1c459cbc35febbfcc81050 2024-11-24T22:16:36,395 INFO [RS:0;bd6592c37bb2:40087-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 8ca644d2cada6dc44b363b7383dcd4fe/C of 8ca644d2cada6dc44b363b7383dcd4fe into 207460af5f1c459cbc35febbfcc81050(size=11.9 K), total size for store is 11.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-24T22:16:36,396 DEBUG [RS:0;bd6592c37bb2:40087-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 8ca644d2cada6dc44b363b7383dcd4fe: 2024-11-24T22:16:36,396 INFO [RS:0;bd6592c37bb2:40087-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732486585339.8ca644d2cada6dc44b363b7383dcd4fe., storeName=8ca644d2cada6dc44b363b7383dcd4fe/C, priority=13, startTime=1732486596352; duration=0sec 2024-11-24T22:16:36,396 DEBUG [RS:0;bd6592c37bb2:40087-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-24T22:16:36,396 DEBUG [RS:0;bd6592c37bb2:40087-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 8ca644d2cada6dc44b363b7383dcd4fe:C 2024-11-24T22:16:36,432 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=40087 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=157 2024-11-24T22:16:36,432 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bd6592c37bb2:0-2 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732486585339.8ca644d2cada6dc44b363b7383dcd4fe. 2024-11-24T22:16:36,433 INFO [RS_FLUSH_OPERATIONS-regionserver/bd6592c37bb2:0-2 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.HRegion(2902): Flushing 8ca644d2cada6dc44b363b7383dcd4fe 3/3 column families, dataSize=100.63 KB heapSize=299.58 KB 2024-11-24T22:16:36,433 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bd6592c37bb2:0-2 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 8ca644d2cada6dc44b363b7383dcd4fe, store=A 2024-11-24T22:16:36,433 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bd6592c37bb2:0-2 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-24T22:16:36,433 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bd6592c37bb2:0-2 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 8ca644d2cada6dc44b363b7383dcd4fe, store=B 2024-11-24T22:16:36,433 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bd6592c37bb2:0-2 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-24T22:16:36,433 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bd6592c37bb2:0-2 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 8ca644d2cada6dc44b363b7383dcd4fe, store=C 2024-11-24T22:16:36,433 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bd6592c37bb2:0-2 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-24T22:16:36,439 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bd6592c37bb2:0-2 {event_type=RS_FLUSH_REGIONS, pid=157}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241124464ea015c92745a988743391450e3d30_8ca644d2cada6dc44b363b7383dcd4fe is 50, key is test_row_0/A:col10/1732486594724/Put/seqid=0 2024-11-24T22:16:36,449 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45929 is added to blk_1073742414_1590 (size=12204) 2024-11-24T22:16:36,763 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40087 {}] regionserver.HRegion(8855): Flush requested on 8ca644d2cada6dc44b363b7383dcd4fe 2024-11-24T22:16:36,763 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732486585339.8ca644d2cada6dc44b363b7383dcd4fe. as already flushing 2024-11-24T22:16:36,813 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40087 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8ca644d2cada6dc44b363b7383dcd4fe, server=bd6592c37bb2,40087,1732486423713 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-24T22:16:36,813 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40087 {}] ipc.CallRunner(138): callId: 26 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49050 deadline: 1732486656809, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8ca644d2cada6dc44b363b7383dcd4fe, server=bd6592c37bb2,40087,1732486423713 2024-11-24T22:16:36,813 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732486585339.8ca644d2cada6dc44b363b7383dcd4fe., hostname=bd6592c37bb2,40087,1732486423713, seqNum=5 , the old value is region=TestAcidGuarantees,,1732486585339.8ca644d2cada6dc44b363b7383dcd4fe., hostname=bd6592c37bb2,40087,1732486423713, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8ca644d2cada6dc44b363b7383dcd4fe, server=bd6592c37bb2,40087,1732486423713 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T22:16:36,814 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732486585339.8ca644d2cada6dc44b363b7383dcd4fe., hostname=bd6592c37bb2,40087,1732486423713, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8ca644d2cada6dc44b363b7383dcd4fe, server=bd6592c37bb2,40087,1732486423713 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T22:16:36,814 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732486585339.8ca644d2cada6dc44b363b7383dcd4fe., hostname=bd6592c37bb2,40087,1732486423713, seqNum=5 because the exception is null or not the one we care about 2024-11-24T22:16:36,850 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bd6592c37bb2:0-2 {event_type=RS_FLUSH_REGIONS, pid=157}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T22:16:36,853 INFO [RS_FLUSH_OPERATIONS-regionserver/bd6592c37bb2:0-2 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241124464ea015c92745a988743391450e3d30_8ca644d2cada6dc44b363b7383dcd4fe to hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241124464ea015c92745a988743391450e3d30_8ca644d2cada6dc44b363b7383dcd4fe 2024-11-24T22:16:36,854 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bd6592c37bb2:0-2 {event_type=RS_FLUSH_REGIONS, pid=157}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/8ca644d2cada6dc44b363b7383dcd4fe/.tmp/A/4c11b34b2e20404193582684f79f36df, store: [table=TestAcidGuarantees family=A region=8ca644d2cada6dc44b363b7383dcd4fe] 2024-11-24T22:16:36,854 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bd6592c37bb2:0-2 {event_type=RS_FLUSH_REGIONS, pid=157}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/8ca644d2cada6dc44b363b7383dcd4fe/.tmp/A/4c11b34b2e20404193582684f79f36df is 175, key is test_row_0/A:col10/1732486594724/Put/seqid=0 2024-11-24T22:16:36,858 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45929 is added to blk_1073742415_1591 (size=31005) 2024-11-24T22:16:36,883 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40087 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8ca644d2cada6dc44b363b7383dcd4fe, server=bd6592c37bb2,40087,1732486423713 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-24T22:16:36,883 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40087 {}] ipc.CallRunner(138): callId: 45 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49060 deadline: 1732486656880, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8ca644d2cada6dc44b363b7383dcd4fe, server=bd6592c37bb2,40087,1732486423713 2024-11-24T22:16:36,883 WARN [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRpcRetryingCaller(177): Call to bd6592c37bb2,40087,1732486423713 for 'test_row_0' in 8ca644d2cada6dc44b363b7383dcd4fe of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2157 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8ca644d2cada6dc44b363b7383dcd4fe, server=bd6592c37bb2,40087,1732486423713 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor44.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8ca644d2cada6dc44b363b7383dcd4fe, server=bd6592c37bb2,40087,1732486423713 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-24T22:16:36,884 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732486585339.8ca644d2cada6dc44b363b7383dcd4fe., hostname=bd6592c37bb2,40087,1732486423713, seqNum=5 , the old value is region=TestAcidGuarantees,,1732486585339.8ca644d2cada6dc44b363b7383dcd4fe., hostname=bd6592c37bb2,40087,1732486423713, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8ca644d2cada6dc44b363b7383dcd4fe, server=bd6592c37bb2,40087,1732486423713 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T22:16:36,884 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732486585339.8ca644d2cada6dc44b363b7383dcd4fe., hostname=bd6592c37bb2,40087,1732486423713, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8ca644d2cada6dc44b363b7383dcd4fe, server=bd6592c37bb2,40087,1732486423713 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T22:16:36,884 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732486585339.8ca644d2cada6dc44b363b7383dcd4fe., hostname=bd6592c37bb2,40087,1732486423713, seqNum=5 because the exception is null or not the one we care about 2024-11-24T22:16:36,892 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40087 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8ca644d2cada6dc44b363b7383dcd4fe, server=bd6592c37bb2,40087,1732486423713 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-24T22:16:36,892 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40087 {}] ipc.CallRunner(138): callId: 54 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49026 deadline: 1732486656891, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8ca644d2cada6dc44b363b7383dcd4fe, server=bd6592c37bb2,40087,1732486423713 2024-11-24T22:16:36,893 WARN [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRpcRetryingCaller(177): Call to bd6592c37bb2,40087,1732486423713 for 'test_row_0' in 8ca644d2cada6dc44b363b7383dcd4fe of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2163 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8ca644d2cada6dc44b363b7383dcd4fe, server=bd6592c37bb2,40087,1732486423713 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor44.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8ca644d2cada6dc44b363b7383dcd4fe, server=bd6592c37bb2,40087,1732486423713 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-24T22:16:36,893 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732486585339.8ca644d2cada6dc44b363b7383dcd4fe., hostname=bd6592c37bb2,40087,1732486423713, seqNum=5 , the old value is region=TestAcidGuarantees,,1732486585339.8ca644d2cada6dc44b363b7383dcd4fe., hostname=bd6592c37bb2,40087,1732486423713, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8ca644d2cada6dc44b363b7383dcd4fe, server=bd6592c37bb2,40087,1732486423713 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T22:16:36,893 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732486585339.8ca644d2cada6dc44b363b7383dcd4fe., hostname=bd6592c37bb2,40087,1732486423713, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8ca644d2cada6dc44b363b7383dcd4fe, server=bd6592c37bb2,40087,1732486423713 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T22:16:36,893 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732486585339.8ca644d2cada6dc44b363b7383dcd4fe., hostname=bd6592c37bb2,40087,1732486423713, seqNum=5 because the exception is null or not the one we care about 2024-11-24T22:16:36,901 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37701 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=156 2024-11-24T22:16:36,924 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40087 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8ca644d2cada6dc44b363b7383dcd4fe, server=bd6592c37bb2,40087,1732486423713 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-24T22:16:36,924 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40087 {}] ipc.CallRunner(138): callId: 27 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49050 deadline: 1732486656921, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8ca644d2cada6dc44b363b7383dcd4fe, server=bd6592c37bb2,40087,1732486423713 2024-11-24T22:16:36,925 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732486585339.8ca644d2cada6dc44b363b7383dcd4fe., hostname=bd6592c37bb2,40087,1732486423713, seqNum=5 , the old value is region=TestAcidGuarantees,,1732486585339.8ca644d2cada6dc44b363b7383dcd4fe., hostname=bd6592c37bb2,40087,1732486423713, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8ca644d2cada6dc44b363b7383dcd4fe, server=bd6592c37bb2,40087,1732486423713 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T22:16:36,925 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732486585339.8ca644d2cada6dc44b363b7383dcd4fe., hostname=bd6592c37bb2,40087,1732486423713, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8ca644d2cada6dc44b363b7383dcd4fe, server=bd6592c37bb2,40087,1732486423713 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T22:16:36,925 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732486585339.8ca644d2cada6dc44b363b7383dcd4fe., hostname=bd6592c37bb2,40087,1732486423713, seqNum=5 because the exception is null or not the one we care about 2024-11-24T22:16:37,131 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40087 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8ca644d2cada6dc44b363b7383dcd4fe, server=bd6592c37bb2,40087,1732486423713 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-24T22:16:37,131 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40087 {}] ipc.CallRunner(138): callId: 28 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49050 deadline: 1732486657130, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8ca644d2cada6dc44b363b7383dcd4fe, server=bd6592c37bb2,40087,1732486423713 2024-11-24T22:16:37,132 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732486585339.8ca644d2cada6dc44b363b7383dcd4fe., hostname=bd6592c37bb2,40087,1732486423713, seqNum=5 , the old value is region=TestAcidGuarantees,,1732486585339.8ca644d2cada6dc44b363b7383dcd4fe., hostname=bd6592c37bb2,40087,1732486423713, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8ca644d2cada6dc44b363b7383dcd4fe, server=bd6592c37bb2,40087,1732486423713 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T22:16:37,132 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732486585339.8ca644d2cada6dc44b363b7383dcd4fe., hostname=bd6592c37bb2,40087,1732486423713, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8ca644d2cada6dc44b363b7383dcd4fe, server=bd6592c37bb2,40087,1732486423713 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T22:16:37,132 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732486585339.8ca644d2cada6dc44b363b7383dcd4fe., hostname=bd6592c37bb2,40087,1732486423713, seqNum=5 because the exception is null or not the one we care about 2024-11-24T22:16:37,258 INFO [RS_FLUSH_OPERATIONS-regionserver/bd6592c37bb2:0-2 {event_type=RS_FLUSH_REGIONS, pid=157}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=134, memsize=33.5 K, hasBloomFilter=true, into tmp file hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/8ca644d2cada6dc44b363b7383dcd4fe/.tmp/A/4c11b34b2e20404193582684f79f36df 2024-11-24T22:16:37,271 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bd6592c37bb2:0-2 {event_type=RS_FLUSH_REGIONS, pid=157}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/8ca644d2cada6dc44b363b7383dcd4fe/.tmp/B/4a689bc4d8984c48b58ed987cdb599df is 50, key is test_row_0/B:col10/1732486594724/Put/seqid=0 2024-11-24T22:16:37,287 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45929 is added to blk_1073742416_1592 (size=12051) 2024-11-24T22:16:37,443 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40087 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8ca644d2cada6dc44b363b7383dcd4fe, server=bd6592c37bb2,40087,1732486423713 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-24T22:16:37,443 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40087 {}] ipc.CallRunner(138): callId: 29 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49050 deadline: 1732486657440, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8ca644d2cada6dc44b363b7383dcd4fe, server=bd6592c37bb2,40087,1732486423713 2024-11-24T22:16:37,444 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732486585339.8ca644d2cada6dc44b363b7383dcd4fe., hostname=bd6592c37bb2,40087,1732486423713, seqNum=5 , the old value is region=TestAcidGuarantees,,1732486585339.8ca644d2cada6dc44b363b7383dcd4fe., hostname=bd6592c37bb2,40087,1732486423713, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8ca644d2cada6dc44b363b7383dcd4fe, server=bd6592c37bb2,40087,1732486423713 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T22:16:37,444 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732486585339.8ca644d2cada6dc44b363b7383dcd4fe., hostname=bd6592c37bb2,40087,1732486423713, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8ca644d2cada6dc44b363b7383dcd4fe, server=bd6592c37bb2,40087,1732486423713 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T22:16:37,444 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732486585339.8ca644d2cada6dc44b363b7383dcd4fe., hostname=bd6592c37bb2,40087,1732486423713, seqNum=5 because the exception is null or not the one we care about 2024-11-24T22:16:37,687 INFO [RS_FLUSH_OPERATIONS-regionserver/bd6592c37bb2:0-2 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=33.54 KB at sequenceid=134 (bloomFilter=true), to=hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/8ca644d2cada6dc44b363b7383dcd4fe/.tmp/B/4a689bc4d8984c48b58ed987cdb599df 2024-11-24T22:16:37,694 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bd6592c37bb2:0-2 {event_type=RS_FLUSH_REGIONS, pid=157}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/8ca644d2cada6dc44b363b7383dcd4fe/.tmp/C/e8487447b7fe43cb9ab44acd74a6b5be is 50, key is test_row_0/C:col10/1732486594724/Put/seqid=0 2024-11-24T22:16:37,701 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45929 is added to blk_1073742417_1593 (size=12051) 2024-11-24T22:16:37,701 INFO [RS_FLUSH_OPERATIONS-regionserver/bd6592c37bb2:0-2 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=33.54 KB at sequenceid=134 (bloomFilter=true), to=hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/8ca644d2cada6dc44b363b7383dcd4fe/.tmp/C/e8487447b7fe43cb9ab44acd74a6b5be 2024-11-24T22:16:37,705 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bd6592c37bb2:0-2 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/8ca644d2cada6dc44b363b7383dcd4fe/.tmp/A/4c11b34b2e20404193582684f79f36df as hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/8ca644d2cada6dc44b363b7383dcd4fe/A/4c11b34b2e20404193582684f79f36df 2024-11-24T22:16:37,708 INFO [RS_FLUSH_OPERATIONS-regionserver/bd6592c37bb2:0-2 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/8ca644d2cada6dc44b363b7383dcd4fe/A/4c11b34b2e20404193582684f79f36df, entries=150, sequenceid=134, filesize=30.3 K 2024-11-24T22:16:37,709 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bd6592c37bb2:0-2 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/8ca644d2cada6dc44b363b7383dcd4fe/.tmp/B/4a689bc4d8984c48b58ed987cdb599df as hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/8ca644d2cada6dc44b363b7383dcd4fe/B/4a689bc4d8984c48b58ed987cdb599df 2024-11-24T22:16:37,712 INFO [RS_FLUSH_OPERATIONS-regionserver/bd6592c37bb2:0-2 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/8ca644d2cada6dc44b363b7383dcd4fe/B/4a689bc4d8984c48b58ed987cdb599df, entries=150, sequenceid=134, filesize=11.8 K 2024-11-24T22:16:37,712 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bd6592c37bb2:0-2 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/8ca644d2cada6dc44b363b7383dcd4fe/.tmp/C/e8487447b7fe43cb9ab44acd74a6b5be as hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/8ca644d2cada6dc44b363b7383dcd4fe/C/e8487447b7fe43cb9ab44acd74a6b5be 2024-11-24T22:16:37,715 INFO [RS_FLUSH_OPERATIONS-regionserver/bd6592c37bb2:0-2 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/8ca644d2cada6dc44b363b7383dcd4fe/C/e8487447b7fe43cb9ab44acd74a6b5be, entries=150, sequenceid=134, filesize=11.8 K 2024-11-24T22:16:37,716 INFO [RS_FLUSH_OPERATIONS-regionserver/bd6592c37bb2:0-2 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.HRegion(3140): Finished flush of dataSize ~100.63 KB/103050, heapSize ~299.53 KB/306720, currentSize=73.80 KB/75570 for 8ca644d2cada6dc44b363b7383dcd4fe in 1284ms, sequenceid=134, compaction requested=false 2024-11-24T22:16:37,716 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bd6592c37bb2:0-2 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.HRegion(2603): Flush status journal for 8ca644d2cada6dc44b363b7383dcd4fe: 2024-11-24T22:16:37,716 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bd6592c37bb2:0-2 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732486585339.8ca644d2cada6dc44b363b7383dcd4fe. 2024-11-24T22:16:37,716 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bd6592c37bb2:0-2 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=157 2024-11-24T22:16:37,716 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37701 {}] master.HMaster(4169): Remote procedure done, pid=157 2024-11-24T22:16:37,719 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=157, resume processing ppid=156 2024-11-24T22:16:37,719 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=157, ppid=156, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 2.9640 sec 2024-11-24T22:16:37,721 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=156, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=156, table=TestAcidGuarantees in 2.9680 sec 2024-11-24T22:16:37,954 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40087 {}] regionserver.HRegion(8855): Flush requested on 8ca644d2cada6dc44b363b7383dcd4fe 2024-11-24T22:16:37,954 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 8ca644d2cada6dc44b363b7383dcd4fe 3/3 column families, dataSize=80.51 KB heapSize=239.81 KB 2024-11-24T22:16:37,954 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 8ca644d2cada6dc44b363b7383dcd4fe, store=A 2024-11-24T22:16:37,954 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-24T22:16:37,954 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 8ca644d2cada6dc44b363b7383dcd4fe, store=B 2024-11-24T22:16:37,954 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-24T22:16:37,954 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 8ca644d2cada6dc44b363b7383dcd4fe, store=C 2024-11-24T22:16:37,954 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-24T22:16:37,969 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241124d793224a549d4d61abfabfb8ba057ae1_8ca644d2cada6dc44b363b7383dcd4fe is 50, key is test_row_0/A:col10/1732486596791/Put/seqid=0 2024-11-24T22:16:37,973 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45929 is added to blk_1073742418_1594 (size=14794) 2024-11-24T22:16:38,028 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40087 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8ca644d2cada6dc44b363b7383dcd4fe, server=bd6592c37bb2,40087,1732486423713 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-24T22:16:38,028 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40087 {}] ipc.CallRunner(138): callId: 45 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49050 deadline: 1732486658024, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8ca644d2cada6dc44b363b7383dcd4fe, server=bd6592c37bb2,40087,1732486423713 2024-11-24T22:16:38,028 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732486585339.8ca644d2cada6dc44b363b7383dcd4fe., hostname=bd6592c37bb2,40087,1732486423713, seqNum=5 , the old value is region=TestAcidGuarantees,,1732486585339.8ca644d2cada6dc44b363b7383dcd4fe., hostname=bd6592c37bb2,40087,1732486423713, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8ca644d2cada6dc44b363b7383dcd4fe, server=bd6592c37bb2,40087,1732486423713 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T22:16:38,029 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732486585339.8ca644d2cada6dc44b363b7383dcd4fe., hostname=bd6592c37bb2,40087,1732486423713, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8ca644d2cada6dc44b363b7383dcd4fe, server=bd6592c37bb2,40087,1732486423713 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T22:16:38,029 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732486585339.8ca644d2cada6dc44b363b7383dcd4fe., hostname=bd6592c37bb2,40087,1732486423713, seqNum=5 because the exception is null or not the one we care about 2024-11-24T22:16:38,132 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40087 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8ca644d2cada6dc44b363b7383dcd4fe, server=bd6592c37bb2,40087,1732486423713 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-24T22:16:38,132 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40087 {}] ipc.CallRunner(138): callId: 46 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49050 deadline: 1732486658130, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8ca644d2cada6dc44b363b7383dcd4fe, server=bd6592c37bb2,40087,1732486423713 2024-11-24T22:16:38,133 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732486585339.8ca644d2cada6dc44b363b7383dcd4fe., hostname=bd6592c37bb2,40087,1732486423713, seqNum=5 , the old value is region=TestAcidGuarantees,,1732486585339.8ca644d2cada6dc44b363b7383dcd4fe., hostname=bd6592c37bb2,40087,1732486423713, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8ca644d2cada6dc44b363b7383dcd4fe, server=bd6592c37bb2,40087,1732486423713 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T22:16:38,133 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732486585339.8ca644d2cada6dc44b363b7383dcd4fe., hostname=bd6592c37bb2,40087,1732486423713, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8ca644d2cada6dc44b363b7383dcd4fe, server=bd6592c37bb2,40087,1732486423713 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T22:16:38,133 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732486585339.8ca644d2cada6dc44b363b7383dcd4fe., hostname=bd6592c37bb2,40087,1732486423713, seqNum=5 because the exception is null or not the one we care about 2024-11-24T22:16:38,343 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40087 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8ca644d2cada6dc44b363b7383dcd4fe, server=bd6592c37bb2,40087,1732486423713 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-24T22:16:38,343 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40087 {}] ipc.CallRunner(138): callId: 47 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49050 deadline: 1732486658341, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8ca644d2cada6dc44b363b7383dcd4fe, server=bd6592c37bb2,40087,1732486423713 2024-11-24T22:16:38,344 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732486585339.8ca644d2cada6dc44b363b7383dcd4fe., hostname=bd6592c37bb2,40087,1732486423713, seqNum=5 , the old value is region=TestAcidGuarantees,,1732486585339.8ca644d2cada6dc44b363b7383dcd4fe., hostname=bd6592c37bb2,40087,1732486423713, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8ca644d2cada6dc44b363b7383dcd4fe, server=bd6592c37bb2,40087,1732486423713 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T22:16:38,344 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732486585339.8ca644d2cada6dc44b363b7383dcd4fe., hostname=bd6592c37bb2,40087,1732486423713, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8ca644d2cada6dc44b363b7383dcd4fe, server=bd6592c37bb2,40087,1732486423713 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T22:16:38,344 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732486585339.8ca644d2cada6dc44b363b7383dcd4fe., hostname=bd6592c37bb2,40087,1732486423713, seqNum=5 because the exception is null or not the one we care about 2024-11-24T22:16:38,374 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T22:16:38,378 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241124d793224a549d4d61abfabfb8ba057ae1_8ca644d2cada6dc44b363b7383dcd4fe to hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241124d793224a549d4d61abfabfb8ba057ae1_8ca644d2cada6dc44b363b7383dcd4fe 2024-11-24T22:16:38,379 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/8ca644d2cada6dc44b363b7383dcd4fe/.tmp/A/7b016023da544892afa2a8f1390bb388, store: [table=TestAcidGuarantees family=A region=8ca644d2cada6dc44b363b7383dcd4fe] 2024-11-24T22:16:38,379 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/8ca644d2cada6dc44b363b7383dcd4fe/.tmp/A/7b016023da544892afa2a8f1390bb388 is 175, key is test_row_0/A:col10/1732486596791/Put/seqid=0 2024-11-24T22:16:38,389 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45929 is added to blk_1073742419_1595 (size=39749) 2024-11-24T22:16:38,652 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40087 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8ca644d2cada6dc44b363b7383dcd4fe, server=bd6592c37bb2,40087,1732486423713 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-24T22:16:38,652 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40087 {}] ipc.CallRunner(138): callId: 48 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49050 deadline: 1732486658650, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8ca644d2cada6dc44b363b7383dcd4fe, server=bd6592c37bb2,40087,1732486423713 2024-11-24T22:16:38,652 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732486585339.8ca644d2cada6dc44b363b7383dcd4fe., hostname=bd6592c37bb2,40087,1732486423713, seqNum=5 , the old value is region=TestAcidGuarantees,,1732486585339.8ca644d2cada6dc44b363b7383dcd4fe., hostname=bd6592c37bb2,40087,1732486423713, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8ca644d2cada6dc44b363b7383dcd4fe, server=bd6592c37bb2,40087,1732486423713 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T22:16:38,652 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732486585339.8ca644d2cada6dc44b363b7383dcd4fe., hostname=bd6592c37bb2,40087,1732486423713, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8ca644d2cada6dc44b363b7383dcd4fe, server=bd6592c37bb2,40087,1732486423713 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T22:16:38,652 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732486585339.8ca644d2cada6dc44b363b7383dcd4fe., hostname=bd6592c37bb2,40087,1732486423713, seqNum=5 because the exception is null or not the one we care about 2024-11-24T22:16:38,789 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=149, memsize=26.8 K, hasBloomFilter=true, into tmp file hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/8ca644d2cada6dc44b363b7383dcd4fe/.tmp/A/7b016023da544892afa2a8f1390bb388 2024-11-24T22:16:38,796 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/8ca644d2cada6dc44b363b7383dcd4fe/.tmp/B/f7f8edc5d3b149d5a11aa186b622f761 is 50, key is test_row_0/B:col10/1732486596791/Put/seqid=0 2024-11-24T22:16:38,803 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45929 is added to blk_1073742420_1596 (size=12151) 2024-11-24T22:16:38,895 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40087 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8ca644d2cada6dc44b363b7383dcd4fe, server=bd6592c37bb2,40087,1732486423713 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-24T22:16:38,895 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40087 {}] ipc.CallRunner(138): callId: 46 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49060 deadline: 1732486658891, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8ca644d2cada6dc44b363b7383dcd4fe, server=bd6592c37bb2,40087,1732486423713 2024-11-24T22:16:38,896 WARN [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRpcRetryingCaller(177): Call to bd6592c37bb2,40087,1732486423713 for 'test_row_0' in 8ca644d2cada6dc44b363b7383dcd4fe of TestAcidGuarantees failed, tries = 7, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 4169 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8ca644d2cada6dc44b363b7383dcd4fe, server=bd6592c37bb2,40087,1732486423713 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor44.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8ca644d2cada6dc44b363b7383dcd4fe, server=bd6592c37bb2,40087,1732486423713 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-24T22:16:38,896 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732486585339.8ca644d2cada6dc44b363b7383dcd4fe., hostname=bd6592c37bb2,40087,1732486423713, seqNum=5 , the old value is region=TestAcidGuarantees,,1732486585339.8ca644d2cada6dc44b363b7383dcd4fe., hostname=bd6592c37bb2,40087,1732486423713, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8ca644d2cada6dc44b363b7383dcd4fe, server=bd6592c37bb2,40087,1732486423713 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T22:16:38,897 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732486585339.8ca644d2cada6dc44b363b7383dcd4fe., hostname=bd6592c37bb2,40087,1732486423713, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8ca644d2cada6dc44b363b7383dcd4fe, server=bd6592c37bb2,40087,1732486423713 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T22:16:38,897 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732486585339.8ca644d2cada6dc44b363b7383dcd4fe., hostname=bd6592c37bb2,40087,1732486423713, seqNum=5 because the exception is null or not the one we care about 2024-11-24T22:16:38,910 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37701 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=156 2024-11-24T22:16:38,911 INFO [RPCClient-NioEventLoopGroup-4-12 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-24T22:16:38,911 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37701 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-24T22:16:38,912 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37701 {}] procedure2.ProcedureExecutor(1139): Stored pid=158, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=158, table=TestAcidGuarantees 2024-11-24T22:16:38,913 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37701 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=158 2024-11-24T22:16:38,913 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=158, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=158, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-24T22:16:38,914 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=158, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=158, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-24T22:16:38,914 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=159, ppid=158, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-24T22:16:38,923 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40087 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8ca644d2cada6dc44b363b7383dcd4fe, server=bd6592c37bb2,40087,1732486423713 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-24T22:16:38,923 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40087 {}] ipc.CallRunner(138): callId: 55 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49026 deadline: 1732486658921, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8ca644d2cada6dc44b363b7383dcd4fe, server=bd6592c37bb2,40087,1732486423713 2024-11-24T22:16:38,924 WARN [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRpcRetryingCaller(177): Call to bd6592c37bb2,40087,1732486423713 for 'test_row_0' in 8ca644d2cada6dc44b363b7383dcd4fe of TestAcidGuarantees failed, tries = 7, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 4194 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8ca644d2cada6dc44b363b7383dcd4fe, server=bd6592c37bb2,40087,1732486423713 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor44.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8ca644d2cada6dc44b363b7383dcd4fe, server=bd6592c37bb2,40087,1732486423713 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-24T22:16:38,924 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732486585339.8ca644d2cada6dc44b363b7383dcd4fe., hostname=bd6592c37bb2,40087,1732486423713, seqNum=5 , the old value is region=TestAcidGuarantees,,1732486585339.8ca644d2cada6dc44b363b7383dcd4fe., hostname=bd6592c37bb2,40087,1732486423713, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8ca644d2cada6dc44b363b7383dcd4fe, server=bd6592c37bb2,40087,1732486423713 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T22:16:38,924 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732486585339.8ca644d2cada6dc44b363b7383dcd4fe., hostname=bd6592c37bb2,40087,1732486423713, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8ca644d2cada6dc44b363b7383dcd4fe, server=bd6592c37bb2,40087,1732486423713 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T22:16:38,924 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732486585339.8ca644d2cada6dc44b363b7383dcd4fe., hostname=bd6592c37bb2,40087,1732486423713, seqNum=5 because the exception is null or not the one we care about 2024-11-24T22:16:39,020 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37701 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=158 2024-11-24T22:16:39,066 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=40087 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=159 2024-11-24T22:16:39,066 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bd6592c37bb2:0-0 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732486585339.8ca644d2cada6dc44b363b7383dcd4fe. 2024-11-24T22:16:39,066 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bd6592c37bb2:0-0 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732486585339.8ca644d2cada6dc44b363b7383dcd4fe. as already flushing 2024-11-24T22:16:39,066 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bd6592c37bb2:0-0 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732486585339.8ca644d2cada6dc44b363b7383dcd4fe. 2024-11-24T22:16:39,066 ERROR [RS_FLUSH_OPERATIONS-regionserver/bd6592c37bb2:0-0 {event_type=RS_FLUSH_REGIONS, pid=159}] handler.RSProcedureHandler(58): pid=159 java.io.IOException: Unable to complete flush {ENCODED => 8ca644d2cada6dc44b363b7383dcd4fe, NAME => 'TestAcidGuarantees,,1732486585339.8ca644d2cada6dc44b363b7383dcd4fe.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T22:16:39,066 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bd6592c37bb2:0-0 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=159 java.io.IOException: Unable to complete flush {ENCODED => 8ca644d2cada6dc44b363b7383dcd4fe, NAME => 'TestAcidGuarantees,,1732486585339.8ca644d2cada6dc44b363b7383dcd4fe.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T22:16:39,067 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37701 {}] master.HMaster(4177): Remote procedure failed, pid=159 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 8ca644d2cada6dc44b363b7383dcd4fe, NAME => 'TestAcidGuarantees,,1732486585339.8ca644d2cada6dc44b363b7383dcd4fe.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 8ca644d2cada6dc44b363b7383dcd4fe, NAME => 'TestAcidGuarantees,,1732486585339.8ca644d2cada6dc44b363b7383dcd4fe.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T22:16:39,163 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40087 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8ca644d2cada6dc44b363b7383dcd4fe, server=bd6592c37bb2,40087,1732486423713 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-24T22:16:39,164 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40087 {}] ipc.CallRunner(138): callId: 49 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49050 deadline: 1732486659161, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8ca644d2cada6dc44b363b7383dcd4fe, server=bd6592c37bb2,40087,1732486423713 2024-11-24T22:16:39,164 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732486585339.8ca644d2cada6dc44b363b7383dcd4fe., hostname=bd6592c37bb2,40087,1732486423713, seqNum=5 , the old value is region=TestAcidGuarantees,,1732486585339.8ca644d2cada6dc44b363b7383dcd4fe., hostname=bd6592c37bb2,40087,1732486423713, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8ca644d2cada6dc44b363b7383dcd4fe, server=bd6592c37bb2,40087,1732486423713 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T22:16:39,164 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732486585339.8ca644d2cada6dc44b363b7383dcd4fe., hostname=bd6592c37bb2,40087,1732486423713, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8ca644d2cada6dc44b363b7383dcd4fe, server=bd6592c37bb2,40087,1732486423713 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T22:16:39,164 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732486585339.8ca644d2cada6dc44b363b7383dcd4fe., hostname=bd6592c37bb2,40087,1732486423713, seqNum=5 because the exception is null or not the one we care about 2024-11-24T22:16:39,204 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=26.84 KB at sequenceid=149 (bloomFilter=true), to=hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/8ca644d2cada6dc44b363b7383dcd4fe/.tmp/B/f7f8edc5d3b149d5a11aa186b622f761 2024-11-24T22:16:39,210 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/8ca644d2cada6dc44b363b7383dcd4fe/.tmp/C/83fb5cbe634d494794af17bfe57a70f5 is 50, key is test_row_0/C:col10/1732486596791/Put/seqid=0 2024-11-24T22:16:39,213 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45929 is added to blk_1073742421_1597 (size=12151) 2024-11-24T22:16:39,218 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=40087 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=159 2024-11-24T22:16:39,218 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bd6592c37bb2:0-1 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732486585339.8ca644d2cada6dc44b363b7383dcd4fe. 2024-11-24T22:16:39,218 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bd6592c37bb2:0-1 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732486585339.8ca644d2cada6dc44b363b7383dcd4fe. as already flushing 2024-11-24T22:16:39,218 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bd6592c37bb2:0-1 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732486585339.8ca644d2cada6dc44b363b7383dcd4fe. 2024-11-24T22:16:39,218 ERROR [RS_FLUSH_OPERATIONS-regionserver/bd6592c37bb2:0-1 {event_type=RS_FLUSH_REGIONS, pid=159}] handler.RSProcedureHandler(58): pid=159 java.io.IOException: Unable to complete flush {ENCODED => 8ca644d2cada6dc44b363b7383dcd4fe, NAME => 'TestAcidGuarantees,,1732486585339.8ca644d2cada6dc44b363b7383dcd4fe.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T22:16:39,219 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bd6592c37bb2:0-1 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=159 java.io.IOException: Unable to complete flush {ENCODED => 8ca644d2cada6dc44b363b7383dcd4fe, NAME => 'TestAcidGuarantees,,1732486585339.8ca644d2cada6dc44b363b7383dcd4fe.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T22:16:39,219 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37701 {}] master.HMaster(4177): Remote procedure failed, pid=159 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 8ca644d2cada6dc44b363b7383dcd4fe, NAME => 'TestAcidGuarantees,,1732486585339.8ca644d2cada6dc44b363b7383dcd4fe.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 8ca644d2cada6dc44b363b7383dcd4fe, NAME => 'TestAcidGuarantees,,1732486585339.8ca644d2cada6dc44b363b7383dcd4fe.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T22:16:39,230 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37701 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=158 2024-11-24T22:16:39,371 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=40087 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=159 2024-11-24T22:16:39,371 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bd6592c37bb2:0-2 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732486585339.8ca644d2cada6dc44b363b7383dcd4fe. 2024-11-24T22:16:39,371 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bd6592c37bb2:0-2 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732486585339.8ca644d2cada6dc44b363b7383dcd4fe. as already flushing 2024-11-24T22:16:39,371 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bd6592c37bb2:0-2 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732486585339.8ca644d2cada6dc44b363b7383dcd4fe. 2024-11-24T22:16:39,371 ERROR [RS_FLUSH_OPERATIONS-regionserver/bd6592c37bb2:0-2 {event_type=RS_FLUSH_REGIONS, pid=159}] handler.RSProcedureHandler(58): pid=159 java.io.IOException: Unable to complete flush {ENCODED => 8ca644d2cada6dc44b363b7383dcd4fe, NAME => 'TestAcidGuarantees,,1732486585339.8ca644d2cada6dc44b363b7383dcd4fe.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T22:16:39,371 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bd6592c37bb2:0-2 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=159 java.io.IOException: Unable to complete flush {ENCODED => 8ca644d2cada6dc44b363b7383dcd4fe, NAME => 'TestAcidGuarantees,,1732486585339.8ca644d2cada6dc44b363b7383dcd4fe.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T22:16:39,372 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37701 {}] master.HMaster(4177): Remote procedure failed, pid=159 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 8ca644d2cada6dc44b363b7383dcd4fe, NAME => 'TestAcidGuarantees,,1732486585339.8ca644d2cada6dc44b363b7383dcd4fe.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 8ca644d2cada6dc44b363b7383dcd4fe, NAME => 'TestAcidGuarantees,,1732486585339.8ca644d2cada6dc44b363b7383dcd4fe.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T22:16:39,524 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=40087 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=159 2024-11-24T22:16:39,524 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bd6592c37bb2:0-0 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732486585339.8ca644d2cada6dc44b363b7383dcd4fe. 2024-11-24T22:16:39,524 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bd6592c37bb2:0-0 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732486585339.8ca644d2cada6dc44b363b7383dcd4fe. as already flushing 2024-11-24T22:16:39,524 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bd6592c37bb2:0-0 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732486585339.8ca644d2cada6dc44b363b7383dcd4fe. 2024-11-24T22:16:39,524 ERROR [RS_FLUSH_OPERATIONS-regionserver/bd6592c37bb2:0-0 {event_type=RS_FLUSH_REGIONS, pid=159}] handler.RSProcedureHandler(58): pid=159 java.io.IOException: Unable to complete flush {ENCODED => 8ca644d2cada6dc44b363b7383dcd4fe, NAME => 'TestAcidGuarantees,,1732486585339.8ca644d2cada6dc44b363b7383dcd4fe.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T22:16:39,524 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bd6592c37bb2:0-0 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=159 java.io.IOException: Unable to complete flush {ENCODED => 8ca644d2cada6dc44b363b7383dcd4fe, NAME => 'TestAcidGuarantees,,1732486585339.8ca644d2cada6dc44b363b7383dcd4fe.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T22:16:39,525 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37701 {}] master.HMaster(4177): Remote procedure failed, pid=159 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 8ca644d2cada6dc44b363b7383dcd4fe, NAME => 'TestAcidGuarantees,,1732486585339.8ca644d2cada6dc44b363b7383dcd4fe.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 8ca644d2cada6dc44b363b7383dcd4fe, NAME => 'TestAcidGuarantees,,1732486585339.8ca644d2cada6dc44b363b7383dcd4fe.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T22:16:39,541 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37701 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=158 2024-11-24T22:16:39,614 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=26.84 KB at sequenceid=149 (bloomFilter=true), to=hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/8ca644d2cada6dc44b363b7383dcd4fe/.tmp/C/83fb5cbe634d494794af17bfe57a70f5 2024-11-24T22:16:39,617 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/8ca644d2cada6dc44b363b7383dcd4fe/.tmp/A/7b016023da544892afa2a8f1390bb388 as hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/8ca644d2cada6dc44b363b7383dcd4fe/A/7b016023da544892afa2a8f1390bb388 2024-11-24T22:16:39,621 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/8ca644d2cada6dc44b363b7383dcd4fe/A/7b016023da544892afa2a8f1390bb388, entries=200, sequenceid=149, filesize=38.8 K 2024-11-24T22:16:39,622 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/8ca644d2cada6dc44b363b7383dcd4fe/.tmp/B/f7f8edc5d3b149d5a11aa186b622f761 as hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/8ca644d2cada6dc44b363b7383dcd4fe/B/f7f8edc5d3b149d5a11aa186b622f761 2024-11-24T22:16:39,625 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/8ca644d2cada6dc44b363b7383dcd4fe/B/f7f8edc5d3b149d5a11aa186b622f761, entries=150, sequenceid=149, filesize=11.9 K 2024-11-24T22:16:39,626 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/8ca644d2cada6dc44b363b7383dcd4fe/.tmp/C/83fb5cbe634d494794af17bfe57a70f5 as hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/8ca644d2cada6dc44b363b7383dcd4fe/C/83fb5cbe634d494794af17bfe57a70f5 2024-11-24T22:16:39,628 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/8ca644d2cada6dc44b363b7383dcd4fe/C/83fb5cbe634d494794af17bfe57a70f5, entries=150, sequenceid=149, filesize=11.9 K 2024-11-24T22:16:39,629 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~80.51 KB/82440, heapSize ~239.77 KB/245520, currentSize=93.93 KB/96180 for 8ca644d2cada6dc44b363b7383dcd4fe in 1675ms, sequenceid=149, compaction requested=true 2024-11-24T22:16:39,629 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 8ca644d2cada6dc44b363b7383dcd4fe: 2024-11-24T22:16:39,629 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 8ca644d2cada6dc44b363b7383dcd4fe:A, priority=-2147483648, current under compaction store size is 1 2024-11-24T22:16:39,629 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-24T22:16:39,629 DEBUG [RS:0;bd6592c37bb2:40087-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-24T22:16:39,629 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 8ca644d2cada6dc44b363b7383dcd4fe:B, priority=-2147483648, current under compaction store size is 2 2024-11-24T22:16:39,629 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-24T22:16:39,629 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 8ca644d2cada6dc44b363b7383dcd4fe:C, priority=-2147483648, current under compaction store size is 3 2024-11-24T22:16:39,629 DEBUG [RS:0;bd6592c37bb2:40087-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-24T22:16:39,629 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-24T22:16:39,630 DEBUG [RS:0;bd6592c37bb2:40087-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 101915 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-24T22:16:39,630 DEBUG [RS:0;bd6592c37bb2:40087-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36409 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-24T22:16:39,630 DEBUG [RS:0;bd6592c37bb2:40087-shortCompactions-0 {}] regionserver.HStore(1541): 8ca644d2cada6dc44b363b7383dcd4fe/A is initiating minor compaction (all files) 2024-11-24T22:16:39,630 DEBUG [RS:0;bd6592c37bb2:40087-longCompactions-0 {}] regionserver.HStore(1541): 8ca644d2cada6dc44b363b7383dcd4fe/B is initiating minor compaction (all files) 2024-11-24T22:16:39,630 INFO [RS:0;bd6592c37bb2:40087-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 8ca644d2cada6dc44b363b7383dcd4fe/A in TestAcidGuarantees,,1732486585339.8ca644d2cada6dc44b363b7383dcd4fe. 2024-11-24T22:16:39,630 INFO [RS:0;bd6592c37bb2:40087-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 8ca644d2cada6dc44b363b7383dcd4fe/B in TestAcidGuarantees,,1732486585339.8ca644d2cada6dc44b363b7383dcd4fe. 2024-11-24T22:16:39,630 INFO [RS:0;bd6592c37bb2:40087-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/8ca644d2cada6dc44b363b7383dcd4fe/A/3cadf84e7037408481c122039cbf1085, hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/8ca644d2cada6dc44b363b7383dcd4fe/A/4c11b34b2e20404193582684f79f36df, hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/8ca644d2cada6dc44b363b7383dcd4fe/A/7b016023da544892afa2a8f1390bb388] into tmpdir=hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/8ca644d2cada6dc44b363b7383dcd4fe/.tmp, totalSize=99.5 K 2024-11-24T22:16:39,630 INFO [RS:0;bd6592c37bb2:40087-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/8ca644d2cada6dc44b363b7383dcd4fe/B/38029836f3024bea8777a5b68b4ad875, hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/8ca644d2cada6dc44b363b7383dcd4fe/B/4a689bc4d8984c48b58ed987cdb599df, hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/8ca644d2cada6dc44b363b7383dcd4fe/B/f7f8edc5d3b149d5a11aa186b622f761] into tmpdir=hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/8ca644d2cada6dc44b363b7383dcd4fe/.tmp, totalSize=35.6 K 2024-11-24T22:16:39,630 INFO [RS:0;bd6592c37bb2:40087-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=13 throughput controller=DefaultCompactionThroughputController [maxThroughput=57.69 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1732486585339.8ca644d2cada6dc44b363b7383dcd4fe. 2024-11-24T22:16:39,630 DEBUG [RS:0;bd6592c37bb2:40087-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1732486585339.8ca644d2cada6dc44b363b7383dcd4fe. files: [hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/8ca644d2cada6dc44b363b7383dcd4fe/A/3cadf84e7037408481c122039cbf1085, hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/8ca644d2cada6dc44b363b7383dcd4fe/A/4c11b34b2e20404193582684f79f36df, hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/8ca644d2cada6dc44b363b7383dcd4fe/A/7b016023da544892afa2a8f1390bb388] 2024-11-24T22:16:39,631 DEBUG [RS:0;bd6592c37bb2:40087-longCompactions-0 {}] compactions.Compactor(225): Compacting 38029836f3024bea8777a5b68b4ad875, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=113, earliestPutTs=1732486594023 2024-11-24T22:16:39,631 DEBUG [RS:0;bd6592c37bb2:40087-shortCompactions-0 {}] compactions.Compactor(225): Compacting 3cadf84e7037408481c122039cbf1085, keycount=150, bloomtype=ROW, size=30.4 K, encoding=NONE, compression=NONE, seqNum=113, earliestPutTs=1732486594023 2024-11-24T22:16:39,631 DEBUG [RS:0;bd6592c37bb2:40087-longCompactions-0 {}] compactions.Compactor(225): Compacting 4a689bc4d8984c48b58ed987cdb599df, keycount=150, bloomtype=ROW, size=11.8 K, encoding=NONE, compression=NONE, seqNum=134, earliestPutTs=1732486594719 2024-11-24T22:16:39,631 DEBUG [RS:0;bd6592c37bb2:40087-shortCompactions-0 {}] compactions.Compactor(225): Compacting 4c11b34b2e20404193582684f79f36df, keycount=150, bloomtype=ROW, size=30.3 K, encoding=NONE, compression=NONE, seqNum=134, earliestPutTs=1732486594719 2024-11-24T22:16:39,631 DEBUG [RS:0;bd6592c37bb2:40087-longCompactions-0 {}] compactions.Compactor(225): Compacting f7f8edc5d3b149d5a11aa186b622f761, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=149, earliestPutTs=1732486596773 2024-11-24T22:16:39,631 DEBUG [RS:0;bd6592c37bb2:40087-shortCompactions-0 {}] compactions.Compactor(225): Compacting 7b016023da544892afa2a8f1390bb388, keycount=200, bloomtype=ROW, size=38.8 K, encoding=NONE, compression=NONE, seqNum=149, earliestPutTs=1732486596773 2024-11-24T22:16:39,636 INFO [RS:0;bd6592c37bb2:40087-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=8ca644d2cada6dc44b363b7383dcd4fe] 2024-11-24T22:16:39,637 INFO [RS:0;bd6592c37bb2:40087-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 8ca644d2cada6dc44b363b7383dcd4fe#B#compaction#506 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 57.69 MB/second 2024-11-24T22:16:39,637 DEBUG [RS:0;bd6592c37bb2:40087-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/8ca644d2cada6dc44b363b7383dcd4fe/.tmp/B/5037fa9e4f23489aa5eede59338d5122 is 50, key is test_row_0/B:col10/1732486596791/Put/seqid=0 2024-11-24T22:16:39,638 DEBUG [RS:0;bd6592c37bb2:40087-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e20241124ada7116777554ac5812a7828563e4135_8ca644d2cada6dc44b363b7383dcd4fe store=[table=TestAcidGuarantees family=A region=8ca644d2cada6dc44b363b7383dcd4fe] 2024-11-24T22:16:39,640 DEBUG [RS:0;bd6592c37bb2:40087-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e20241124ada7116777554ac5812a7828563e4135_8ca644d2cada6dc44b363b7383dcd4fe, store=[table=TestAcidGuarantees family=A region=8ca644d2cada6dc44b363b7383dcd4fe] 2024-11-24T22:16:39,640 DEBUG [RS:0;bd6592c37bb2:40087-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241124ada7116777554ac5812a7828563e4135_8ca644d2cada6dc44b363b7383dcd4fe because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=8ca644d2cada6dc44b363b7383dcd4fe] 2024-11-24T22:16:39,643 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45929 is added to blk_1073742422_1598 (size=12459) 2024-11-24T22:16:39,644 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45929 is added to blk_1073742423_1599 (size=4469) 2024-11-24T22:16:39,676 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=40087 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=159 2024-11-24T22:16:39,676 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bd6592c37bb2:0-1 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732486585339.8ca644d2cada6dc44b363b7383dcd4fe. 2024-11-24T22:16:39,676 INFO [RS_FLUSH_OPERATIONS-regionserver/bd6592c37bb2:0-1 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.HRegion(2902): Flushing 8ca644d2cada6dc44b363b7383dcd4fe 3/3 column families, dataSize=93.93 KB heapSize=279.66 KB 2024-11-24T22:16:39,677 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bd6592c37bb2:0-1 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 8ca644d2cada6dc44b363b7383dcd4fe, store=A 2024-11-24T22:16:39,677 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bd6592c37bb2:0-1 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-24T22:16:39,677 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bd6592c37bb2:0-1 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 8ca644d2cada6dc44b363b7383dcd4fe, store=B 2024-11-24T22:16:39,677 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bd6592c37bb2:0-1 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-24T22:16:39,677 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bd6592c37bb2:0-1 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 8ca644d2cada6dc44b363b7383dcd4fe, store=C 2024-11-24T22:16:39,677 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bd6592c37bb2:0-1 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-24T22:16:39,682 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bd6592c37bb2:0-1 {event_type=RS_FLUSH_REGIONS, pid=159}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411244c657d2b8c0349f29eecc4bb03459c63_8ca644d2cada6dc44b363b7383dcd4fe is 50, key is test_row_0/A:col10/1732486598022/Put/seqid=0 2024-11-24T22:16:39,685 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45929 is added to blk_1073742424_1600 (size=12304) 2024-11-24T22:16:40,045 INFO [RS:0;bd6592c37bb2:40087-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 8ca644d2cada6dc44b363b7383dcd4fe#A#compaction#507 average throughput is 0.06 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 57.69 MB/second 2024-11-24T22:16:40,045 DEBUG [RS:0;bd6592c37bb2:40087-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/8ca644d2cada6dc44b363b7383dcd4fe/.tmp/A/b722343fe5fc48559de2c4b094712e51 is 175, key is test_row_0/A:col10/1732486596791/Put/seqid=0 2024-11-24T22:16:40,047 DEBUG [RS:0;bd6592c37bb2:40087-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/8ca644d2cada6dc44b363b7383dcd4fe/.tmp/B/5037fa9e4f23489aa5eede59338d5122 as hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/8ca644d2cada6dc44b363b7383dcd4fe/B/5037fa9e4f23489aa5eede59338d5122 2024-11-24T22:16:40,050 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45929 is added to blk_1073742425_1601 (size=31413) 2024-11-24T22:16:40,051 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37701 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=158 2024-11-24T22:16:40,052 INFO [RS:0;bd6592c37bb2:40087-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 8ca644d2cada6dc44b363b7383dcd4fe/B of 8ca644d2cada6dc44b363b7383dcd4fe into 5037fa9e4f23489aa5eede59338d5122(size=12.2 K), total size for store is 12.2 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-24T22:16:40,052 DEBUG [RS:0;bd6592c37bb2:40087-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 8ca644d2cada6dc44b363b7383dcd4fe: 2024-11-24T22:16:40,052 INFO [RS:0;bd6592c37bb2:40087-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732486585339.8ca644d2cada6dc44b363b7383dcd4fe., storeName=8ca644d2cada6dc44b363b7383dcd4fe/B, priority=13, startTime=1732486599629; duration=0sec 2024-11-24T22:16:40,052 DEBUG [RS:0;bd6592c37bb2:40087-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-24T22:16:40,052 DEBUG [RS:0;bd6592c37bb2:40087-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 8ca644d2cada6dc44b363b7383dcd4fe:B 2024-11-24T22:16:40,052 DEBUG [RS:0;bd6592c37bb2:40087-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-24T22:16:40,053 DEBUG [RS:0;bd6592c37bb2:40087-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36409 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-24T22:16:40,053 DEBUG [RS:0;bd6592c37bb2:40087-longCompactions-0 {}] regionserver.HStore(1541): 8ca644d2cada6dc44b363b7383dcd4fe/C is initiating minor compaction (all files) 2024-11-24T22:16:40,053 INFO [RS:0;bd6592c37bb2:40087-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 8ca644d2cada6dc44b363b7383dcd4fe/C in TestAcidGuarantees,,1732486585339.8ca644d2cada6dc44b363b7383dcd4fe. 2024-11-24T22:16:40,053 INFO [RS:0;bd6592c37bb2:40087-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/8ca644d2cada6dc44b363b7383dcd4fe/C/207460af5f1c459cbc35febbfcc81050, hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/8ca644d2cada6dc44b363b7383dcd4fe/C/e8487447b7fe43cb9ab44acd74a6b5be, hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/8ca644d2cada6dc44b363b7383dcd4fe/C/83fb5cbe634d494794af17bfe57a70f5] into tmpdir=hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/8ca644d2cada6dc44b363b7383dcd4fe/.tmp, totalSize=35.6 K 2024-11-24T22:16:40,053 DEBUG [RS:0;bd6592c37bb2:40087-longCompactions-0 {}] compactions.Compactor(225): Compacting 207460af5f1c459cbc35febbfcc81050, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=113, earliestPutTs=1732486594023 2024-11-24T22:16:40,053 DEBUG [RS:0;bd6592c37bb2:40087-longCompactions-0 {}] compactions.Compactor(225): Compacting e8487447b7fe43cb9ab44acd74a6b5be, keycount=150, bloomtype=ROW, size=11.8 K, encoding=NONE, compression=NONE, seqNum=134, earliestPutTs=1732486594719 2024-11-24T22:16:40,054 DEBUG [RS:0;bd6592c37bb2:40087-longCompactions-0 {}] compactions.Compactor(225): Compacting 83fb5cbe634d494794af17bfe57a70f5, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=149, earliestPutTs=1732486596773 2024-11-24T22:16:40,059 INFO [RS:0;bd6592c37bb2:40087-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 8ca644d2cada6dc44b363b7383dcd4fe#C#compaction#509 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 57.69 MB/second 2024-11-24T22:16:40,059 DEBUG [RS:0;bd6592c37bb2:40087-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/8ca644d2cada6dc44b363b7383dcd4fe/.tmp/C/4c146cfb566b43de8d7bda68c062baa8 is 50, key is test_row_0/C:col10/1732486596791/Put/seqid=0 2024-11-24T22:16:40,069 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45929 is added to blk_1073742426_1602 (size=12459) 2024-11-24T22:16:40,073 DEBUG [RS:0;bd6592c37bb2:40087-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/8ca644d2cada6dc44b363b7383dcd4fe/.tmp/C/4c146cfb566b43de8d7bda68c062baa8 as hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/8ca644d2cada6dc44b363b7383dcd4fe/C/4c146cfb566b43de8d7bda68c062baa8 2024-11-24T22:16:40,077 INFO [RS:0;bd6592c37bb2:40087-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 8ca644d2cada6dc44b363b7383dcd4fe/C of 8ca644d2cada6dc44b363b7383dcd4fe into 4c146cfb566b43de8d7bda68c062baa8(size=12.2 K), total size for store is 12.2 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-24T22:16:40,077 DEBUG [RS:0;bd6592c37bb2:40087-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 8ca644d2cada6dc44b363b7383dcd4fe: 2024-11-24T22:16:40,077 INFO [RS:0;bd6592c37bb2:40087-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732486585339.8ca644d2cada6dc44b363b7383dcd4fe., storeName=8ca644d2cada6dc44b363b7383dcd4fe/C, priority=13, startTime=1732486599629; duration=0sec 2024-11-24T22:16:40,077 DEBUG [RS:0;bd6592c37bb2:40087-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-24T22:16:40,077 DEBUG [RS:0;bd6592c37bb2:40087-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 8ca644d2cada6dc44b363b7383dcd4fe:C 2024-11-24T22:16:40,086 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bd6592c37bb2:0-1 {event_type=RS_FLUSH_REGIONS, pid=159}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T22:16:40,089 INFO [RS_FLUSH_OPERATIONS-regionserver/bd6592c37bb2:0-1 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411244c657d2b8c0349f29eecc4bb03459c63_8ca644d2cada6dc44b363b7383dcd4fe to hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411244c657d2b8c0349f29eecc4bb03459c63_8ca644d2cada6dc44b363b7383dcd4fe 2024-11-24T22:16:40,089 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bd6592c37bb2:0-1 {event_type=RS_FLUSH_REGIONS, pid=159}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/8ca644d2cada6dc44b363b7383dcd4fe/.tmp/A/c679a2427e9f4fcea97a02498a755dde, store: [table=TestAcidGuarantees family=A region=8ca644d2cada6dc44b363b7383dcd4fe] 2024-11-24T22:16:40,090 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bd6592c37bb2:0-1 {event_type=RS_FLUSH_REGIONS, pid=159}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/8ca644d2cada6dc44b363b7383dcd4fe/.tmp/A/c679a2427e9f4fcea97a02498a755dde is 175, key is test_row_0/A:col10/1732486598022/Put/seqid=0 2024-11-24T22:16:40,097 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45929 is added to blk_1073742427_1603 (size=31105) 2024-11-24T22:16:40,098 INFO [RS_FLUSH_OPERATIONS-regionserver/bd6592c37bb2:0-1 {event_type=RS_FLUSH_REGIONS, pid=159}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=166, memsize=31.3 K, hasBloomFilter=true, into tmp file hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/8ca644d2cada6dc44b363b7383dcd4fe/.tmp/A/c679a2427e9f4fcea97a02498a755dde 2024-11-24T22:16:40,104 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bd6592c37bb2:0-1 {event_type=RS_FLUSH_REGIONS, pid=159}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/8ca644d2cada6dc44b363b7383dcd4fe/.tmp/B/31584a242fc64dfcb888a07f8cbddd9f is 50, key is test_row_0/B:col10/1732486598022/Put/seqid=0 2024-11-24T22:16:40,110 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45929 is added to blk_1073742428_1604 (size=12151) 2024-11-24T22:16:40,110 INFO [RS_FLUSH_OPERATIONS-regionserver/bd6592c37bb2:0-1 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=31.31 KB at sequenceid=166 (bloomFilter=true), to=hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/8ca644d2cada6dc44b363b7383dcd4fe/.tmp/B/31584a242fc64dfcb888a07f8cbddd9f 2024-11-24T22:16:40,118 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bd6592c37bb2:0-1 {event_type=RS_FLUSH_REGIONS, pid=159}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/8ca644d2cada6dc44b363b7383dcd4fe/.tmp/C/0fec38d8c31a41a8a5233d65261f9229 is 50, key is test_row_0/C:col10/1732486598022/Put/seqid=0 2024-11-24T22:16:40,120 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45929 is added to blk_1073742429_1605 (size=12151) 2024-11-24T22:16:40,134 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40087 {}] regionserver.HRegion(8855): Flush requested on 8ca644d2cada6dc44b363b7383dcd4fe 2024-11-24T22:16:40,134 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732486585339.8ca644d2cada6dc44b363b7383dcd4fe. as already flushing 2024-11-24T22:16:40,185 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40087 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8ca644d2cada6dc44b363b7383dcd4fe, server=bd6592c37bb2,40087,1732486423713 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-24T22:16:40,186 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40087 {}] ipc.CallRunner(138): callId: 46 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48986 deadline: 1732486660179, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8ca644d2cada6dc44b363b7383dcd4fe, server=bd6592c37bb2,40087,1732486423713 2024-11-24T22:16:40,186 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732486585339.8ca644d2cada6dc44b363b7383dcd4fe., hostname=bd6592c37bb2,40087,1732486423713, seqNum=5 , the old value is region=TestAcidGuarantees,,1732486585339.8ca644d2cada6dc44b363b7383dcd4fe., hostname=bd6592c37bb2,40087,1732486423713, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8ca644d2cada6dc44b363b7383dcd4fe, server=bd6592c37bb2,40087,1732486423713 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T22:16:40,186 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40087 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8ca644d2cada6dc44b363b7383dcd4fe, server=bd6592c37bb2,40087,1732486423713 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-24T22:16:40,186 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732486585339.8ca644d2cada6dc44b363b7383dcd4fe., hostname=bd6592c37bb2,40087,1732486423713, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8ca644d2cada6dc44b363b7383dcd4fe, server=bd6592c37bb2,40087,1732486423713 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T22:16:40,186 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732486585339.8ca644d2cada6dc44b363b7383dcd4fe., hostname=bd6592c37bb2,40087,1732486423713, seqNum=5 because the exception is null or not the one we care about 2024-11-24T22:16:40,186 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40087 {}] ipc.CallRunner(138): callId: 44 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49002 deadline: 1732486660180, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8ca644d2cada6dc44b363b7383dcd4fe, server=bd6592c37bb2,40087,1732486423713 2024-11-24T22:16:40,186 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732486585339.8ca644d2cada6dc44b363b7383dcd4fe., hostname=bd6592c37bb2,40087,1732486423713, seqNum=5 , the old value is region=TestAcidGuarantees,,1732486585339.8ca644d2cada6dc44b363b7383dcd4fe., hostname=bd6592c37bb2,40087,1732486423713, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8ca644d2cada6dc44b363b7383dcd4fe, server=bd6592c37bb2,40087,1732486423713 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T22:16:40,186 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732486585339.8ca644d2cada6dc44b363b7383dcd4fe., hostname=bd6592c37bb2,40087,1732486423713, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8ca644d2cada6dc44b363b7383dcd4fe, server=bd6592c37bb2,40087,1732486423713 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T22:16:40,186 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732486585339.8ca644d2cada6dc44b363b7383dcd4fe., hostname=bd6592c37bb2,40087,1732486423713, seqNum=5 because the exception is null or not the one we care about 2024-11-24T22:16:40,188 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40087 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8ca644d2cada6dc44b363b7383dcd4fe, server=bd6592c37bb2,40087,1732486423713 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-24T22:16:40,188 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40087 {}] ipc.CallRunner(138): callId: 52 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49050 deadline: 1732486660184, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8ca644d2cada6dc44b363b7383dcd4fe, server=bd6592c37bb2,40087,1732486423713 2024-11-24T22:16:40,189 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732486585339.8ca644d2cada6dc44b363b7383dcd4fe., hostname=bd6592c37bb2,40087,1732486423713, seqNum=5 , the old value is region=TestAcidGuarantees,,1732486585339.8ca644d2cada6dc44b363b7383dcd4fe., hostname=bd6592c37bb2,40087,1732486423713, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8ca644d2cada6dc44b363b7383dcd4fe, server=bd6592c37bb2,40087,1732486423713 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T22:16:40,189 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732486585339.8ca644d2cada6dc44b363b7383dcd4fe., hostname=bd6592c37bb2,40087,1732486423713, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8ca644d2cada6dc44b363b7383dcd4fe, server=bd6592c37bb2,40087,1732486423713 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T22:16:40,189 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732486585339.8ca644d2cada6dc44b363b7383dcd4fe., hostname=bd6592c37bb2,40087,1732486423713, seqNum=5 because the exception is null or not the one we care about 2024-11-24T22:16:40,295 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40087 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8ca644d2cada6dc44b363b7383dcd4fe, server=bd6592c37bb2,40087,1732486423713 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-24T22:16:40,295 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40087 {}] ipc.CallRunner(138): callId: 47 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48986 deadline: 1732486660291, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8ca644d2cada6dc44b363b7383dcd4fe, server=bd6592c37bb2,40087,1732486423713 2024-11-24T22:16:40,295 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40087 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8ca644d2cada6dc44b363b7383dcd4fe, server=bd6592c37bb2,40087,1732486423713 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-24T22:16:40,295 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732486585339.8ca644d2cada6dc44b363b7383dcd4fe., hostname=bd6592c37bb2,40087,1732486423713, seqNum=5 , the old value is region=TestAcidGuarantees,,1732486585339.8ca644d2cada6dc44b363b7383dcd4fe., hostname=bd6592c37bb2,40087,1732486423713, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8ca644d2cada6dc44b363b7383dcd4fe, server=bd6592c37bb2,40087,1732486423713 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T22:16:40,295 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732486585339.8ca644d2cada6dc44b363b7383dcd4fe., hostname=bd6592c37bb2,40087,1732486423713, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8ca644d2cada6dc44b363b7383dcd4fe, server=bd6592c37bb2,40087,1732486423713 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T22:16:40,295 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732486585339.8ca644d2cada6dc44b363b7383dcd4fe., hostname=bd6592c37bb2,40087,1732486423713, seqNum=5 because the exception is null or not the one we care about 2024-11-24T22:16:40,295 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40087 {}] ipc.CallRunner(138): callId: 45 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49002 deadline: 1732486660291, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8ca644d2cada6dc44b363b7383dcd4fe, server=bd6592c37bb2,40087,1732486423713 2024-11-24T22:16:40,296 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40087 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8ca644d2cada6dc44b363b7383dcd4fe, server=bd6592c37bb2,40087,1732486423713 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-24T22:16:40,296 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732486585339.8ca644d2cada6dc44b363b7383dcd4fe., hostname=bd6592c37bb2,40087,1732486423713, seqNum=5 , the old value is region=TestAcidGuarantees,,1732486585339.8ca644d2cada6dc44b363b7383dcd4fe., hostname=bd6592c37bb2,40087,1732486423713, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8ca644d2cada6dc44b363b7383dcd4fe, server=bd6592c37bb2,40087,1732486423713 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T22:16:40,296 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40087 {}] ipc.CallRunner(138): callId: 53 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49050 deadline: 1732486660291, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8ca644d2cada6dc44b363b7383dcd4fe, server=bd6592c37bb2,40087,1732486423713 2024-11-24T22:16:40,296 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732486585339.8ca644d2cada6dc44b363b7383dcd4fe., hostname=bd6592c37bb2,40087,1732486423713, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8ca644d2cada6dc44b363b7383dcd4fe, server=bd6592c37bb2,40087,1732486423713 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T22:16:40,296 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732486585339.8ca644d2cada6dc44b363b7383dcd4fe., hostname=bd6592c37bb2,40087,1732486423713, seqNum=5 because the exception is null or not the one we care about 2024-11-24T22:16:40,296 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732486585339.8ca644d2cada6dc44b363b7383dcd4fe., hostname=bd6592c37bb2,40087,1732486423713, seqNum=5 , the old value is region=TestAcidGuarantees,,1732486585339.8ca644d2cada6dc44b363b7383dcd4fe., hostname=bd6592c37bb2,40087,1732486423713, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8ca644d2cada6dc44b363b7383dcd4fe, server=bd6592c37bb2,40087,1732486423713 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T22:16:40,296 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732486585339.8ca644d2cada6dc44b363b7383dcd4fe., hostname=bd6592c37bb2,40087,1732486423713, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8ca644d2cada6dc44b363b7383dcd4fe, server=bd6592c37bb2,40087,1732486423713 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T22:16:40,296 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732486585339.8ca644d2cada6dc44b363b7383dcd4fe., hostname=bd6592c37bb2,40087,1732486423713, seqNum=5 because the exception is null or not the one we care about 2024-11-24T22:16:40,454 DEBUG [RS:0;bd6592c37bb2:40087-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/8ca644d2cada6dc44b363b7383dcd4fe/.tmp/A/b722343fe5fc48559de2c4b094712e51 as hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/8ca644d2cada6dc44b363b7383dcd4fe/A/b722343fe5fc48559de2c4b094712e51 2024-11-24T22:16:40,458 INFO [RS:0;bd6592c37bb2:40087-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 8ca644d2cada6dc44b363b7383dcd4fe/A of 8ca644d2cada6dc44b363b7383dcd4fe into b722343fe5fc48559de2c4b094712e51(size=30.7 K), total size for store is 30.7 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-24T22:16:40,458 DEBUG [RS:0;bd6592c37bb2:40087-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 8ca644d2cada6dc44b363b7383dcd4fe: 2024-11-24T22:16:40,458 INFO [RS:0;bd6592c37bb2:40087-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732486585339.8ca644d2cada6dc44b363b7383dcd4fe., storeName=8ca644d2cada6dc44b363b7383dcd4fe/A, priority=13, startTime=1732486599629; duration=0sec 2024-11-24T22:16:40,458 DEBUG [RS:0;bd6592c37bb2:40087-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-24T22:16:40,458 DEBUG [RS:0;bd6592c37bb2:40087-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 8ca644d2cada6dc44b363b7383dcd4fe:A 2024-11-24T22:16:40,504 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40087 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8ca644d2cada6dc44b363b7383dcd4fe, server=bd6592c37bb2,40087,1732486423713 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-24T22:16:40,504 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40087 {}] ipc.CallRunner(138): callId: 46 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49002 deadline: 1732486660501, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8ca644d2cada6dc44b363b7383dcd4fe, server=bd6592c37bb2,40087,1732486423713 2024-11-24T22:16:40,504 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732486585339.8ca644d2cada6dc44b363b7383dcd4fe., hostname=bd6592c37bb2,40087,1732486423713, seqNum=5 , the old value is region=TestAcidGuarantees,,1732486585339.8ca644d2cada6dc44b363b7383dcd4fe., hostname=bd6592c37bb2,40087,1732486423713, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8ca644d2cada6dc44b363b7383dcd4fe, server=bd6592c37bb2,40087,1732486423713 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T22:16:40,504 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40087 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8ca644d2cada6dc44b363b7383dcd4fe, server=bd6592c37bb2,40087,1732486423713 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-24T22:16:40,504 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732486585339.8ca644d2cada6dc44b363b7383dcd4fe., hostname=bd6592c37bb2,40087,1732486423713, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8ca644d2cada6dc44b363b7383dcd4fe, server=bd6592c37bb2,40087,1732486423713 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T22:16:40,504 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732486585339.8ca644d2cada6dc44b363b7383dcd4fe., hostname=bd6592c37bb2,40087,1732486423713, seqNum=5 because the exception is null or not the one we care about 2024-11-24T22:16:40,504 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40087 {}] ipc.CallRunner(138): callId: 48 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48986 deadline: 1732486660501, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8ca644d2cada6dc44b363b7383dcd4fe, server=bd6592c37bb2,40087,1732486423713 2024-11-24T22:16:40,505 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732486585339.8ca644d2cada6dc44b363b7383dcd4fe., hostname=bd6592c37bb2,40087,1732486423713, seqNum=5 , the old value is region=TestAcidGuarantees,,1732486585339.8ca644d2cada6dc44b363b7383dcd4fe., hostname=bd6592c37bb2,40087,1732486423713, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8ca644d2cada6dc44b363b7383dcd4fe, server=bd6592c37bb2,40087,1732486423713 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T22:16:40,505 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732486585339.8ca644d2cada6dc44b363b7383dcd4fe., hostname=bd6592c37bb2,40087,1732486423713, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8ca644d2cada6dc44b363b7383dcd4fe, server=bd6592c37bb2,40087,1732486423713 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T22:16:40,505 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40087 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8ca644d2cada6dc44b363b7383dcd4fe, server=bd6592c37bb2,40087,1732486423713 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-24T22:16:40,505 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732486585339.8ca644d2cada6dc44b363b7383dcd4fe., hostname=bd6592c37bb2,40087,1732486423713, seqNum=5 because the exception is null or not the one we care about 2024-11-24T22:16:40,505 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40087 {}] ipc.CallRunner(138): callId: 54 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49050 deadline: 1732486660501, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8ca644d2cada6dc44b363b7383dcd4fe, server=bd6592c37bb2,40087,1732486423713 2024-11-24T22:16:40,505 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732486585339.8ca644d2cada6dc44b363b7383dcd4fe., hostname=bd6592c37bb2,40087,1732486423713, seqNum=5 , the old value is region=TestAcidGuarantees,,1732486585339.8ca644d2cada6dc44b363b7383dcd4fe., hostname=bd6592c37bb2,40087,1732486423713, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8ca644d2cada6dc44b363b7383dcd4fe, server=bd6592c37bb2,40087,1732486423713 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T22:16:40,505 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732486585339.8ca644d2cada6dc44b363b7383dcd4fe., hostname=bd6592c37bb2,40087,1732486423713, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8ca644d2cada6dc44b363b7383dcd4fe, server=bd6592c37bb2,40087,1732486423713 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T22:16:40,505 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732486585339.8ca644d2cada6dc44b363b7383dcd4fe., hostname=bd6592c37bb2,40087,1732486423713, seqNum=5 because the exception is null or not the one we care about 2024-11-24T22:16:40,521 INFO [RS_FLUSH_OPERATIONS-regionserver/bd6592c37bb2:0-1 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=31.31 KB at sequenceid=166 (bloomFilter=true), to=hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/8ca644d2cada6dc44b363b7383dcd4fe/.tmp/C/0fec38d8c31a41a8a5233d65261f9229 2024-11-24T22:16:40,525 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bd6592c37bb2:0-1 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/8ca644d2cada6dc44b363b7383dcd4fe/.tmp/A/c679a2427e9f4fcea97a02498a755dde as hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/8ca644d2cada6dc44b363b7383dcd4fe/A/c679a2427e9f4fcea97a02498a755dde 2024-11-24T22:16:40,528 INFO [RS_FLUSH_OPERATIONS-regionserver/bd6592c37bb2:0-1 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/8ca644d2cada6dc44b363b7383dcd4fe/A/c679a2427e9f4fcea97a02498a755dde, entries=150, sequenceid=166, filesize=30.4 K 2024-11-24T22:16:40,529 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bd6592c37bb2:0-1 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/8ca644d2cada6dc44b363b7383dcd4fe/.tmp/B/31584a242fc64dfcb888a07f8cbddd9f as hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/8ca644d2cada6dc44b363b7383dcd4fe/B/31584a242fc64dfcb888a07f8cbddd9f 2024-11-24T22:16:40,532 INFO [RS_FLUSH_OPERATIONS-regionserver/bd6592c37bb2:0-1 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/8ca644d2cada6dc44b363b7383dcd4fe/B/31584a242fc64dfcb888a07f8cbddd9f, entries=150, sequenceid=166, filesize=11.9 K 2024-11-24T22:16:40,533 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bd6592c37bb2:0-1 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/8ca644d2cada6dc44b363b7383dcd4fe/.tmp/C/0fec38d8c31a41a8a5233d65261f9229 as hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/8ca644d2cada6dc44b363b7383dcd4fe/C/0fec38d8c31a41a8a5233d65261f9229 2024-11-24T22:16:40,536 INFO [RS_FLUSH_OPERATIONS-regionserver/bd6592c37bb2:0-1 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/8ca644d2cada6dc44b363b7383dcd4fe/C/0fec38d8c31a41a8a5233d65261f9229, entries=150, sequenceid=166, filesize=11.9 K 2024-11-24T22:16:40,537 INFO [RS_FLUSH_OPERATIONS-regionserver/bd6592c37bb2:0-1 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.HRegion(3140): Finished flush of dataSize ~93.93 KB/96180, heapSize ~279.61 KB/286320, currentSize=80.51 KB/82440 for 8ca644d2cada6dc44b363b7383dcd4fe in 861ms, sequenceid=166, compaction requested=false 2024-11-24T22:16:40,537 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bd6592c37bb2:0-1 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.HRegion(2603): Flush status journal for 8ca644d2cada6dc44b363b7383dcd4fe: 2024-11-24T22:16:40,537 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bd6592c37bb2:0-1 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732486585339.8ca644d2cada6dc44b363b7383dcd4fe. 2024-11-24T22:16:40,537 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bd6592c37bb2:0-1 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=159 2024-11-24T22:16:40,537 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37701 {}] master.HMaster(4169): Remote procedure done, pid=159 2024-11-24T22:16:40,539 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=159, resume processing ppid=158 2024-11-24T22:16:40,539 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=159, ppid=158, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.6240 sec 2024-11-24T22:16:40,541 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=158, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=158, table=TestAcidGuarantees in 1.6290 sec 2024-11-24T22:16:40,813 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40087 {}] regionserver.HRegion(8855): Flush requested on 8ca644d2cada6dc44b363b7383dcd4fe 2024-11-24T22:16:40,813 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 8ca644d2cada6dc44b363b7383dcd4fe 3/3 column families, dataSize=87.22 KB heapSize=259.73 KB 2024-11-24T22:16:40,814 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 8ca644d2cada6dc44b363b7383dcd4fe, store=A 2024-11-24T22:16:40,814 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-24T22:16:40,814 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 8ca644d2cada6dc44b363b7383dcd4fe, store=B 2024-11-24T22:16:40,814 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-24T22:16:40,814 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 8ca644d2cada6dc44b363b7383dcd4fe, store=C 2024-11-24T22:16:40,814 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-24T22:16:40,821 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241124df325bb5e0354ee5be9c82bf6a0b1763_8ca644d2cada6dc44b363b7383dcd4fe is 50, key is test_row_0/A:col10/1732486600150/Put/seqid=0 2024-11-24T22:16:40,828 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45929 is added to blk_1073742430_1606 (size=17284) 2024-11-24T22:16:40,837 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40087 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8ca644d2cada6dc44b363b7383dcd4fe, server=bd6592c37bb2,40087,1732486423713 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-24T22:16:40,837 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40087 {}] ipc.CallRunner(138): callId: 59 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49050 deadline: 1732486660832, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8ca644d2cada6dc44b363b7383dcd4fe, server=bd6592c37bb2,40087,1732486423713 2024-11-24T22:16:40,838 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40087 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8ca644d2cada6dc44b363b7383dcd4fe, server=bd6592c37bb2,40087,1732486423713 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-24T22:16:40,838 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732486585339.8ca644d2cada6dc44b363b7383dcd4fe., hostname=bd6592c37bb2,40087,1732486423713, seqNum=5 , the old value is region=TestAcidGuarantees,,1732486585339.8ca644d2cada6dc44b363b7383dcd4fe., hostname=bd6592c37bb2,40087,1732486423713, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8ca644d2cada6dc44b363b7383dcd4fe, server=bd6592c37bb2,40087,1732486423713 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T22:16:40,838 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732486585339.8ca644d2cada6dc44b363b7383dcd4fe., hostname=bd6592c37bb2,40087,1732486423713, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8ca644d2cada6dc44b363b7383dcd4fe, server=bd6592c37bb2,40087,1732486423713 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T22:16:40,838 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40087 {}] ipc.CallRunner(138): callId: 52 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49002 deadline: 1732486660833, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8ca644d2cada6dc44b363b7383dcd4fe, server=bd6592c37bb2,40087,1732486423713 2024-11-24T22:16:40,838 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732486585339.8ca644d2cada6dc44b363b7383dcd4fe., hostname=bd6592c37bb2,40087,1732486423713, seqNum=5 because the exception is null or not the one we care about 2024-11-24T22:16:40,838 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732486585339.8ca644d2cada6dc44b363b7383dcd4fe., hostname=bd6592c37bb2,40087,1732486423713, seqNum=5 , the old value is region=TestAcidGuarantees,,1732486585339.8ca644d2cada6dc44b363b7383dcd4fe., hostname=bd6592c37bb2,40087,1732486423713, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8ca644d2cada6dc44b363b7383dcd4fe, server=bd6592c37bb2,40087,1732486423713 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T22:16:40,838 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732486585339.8ca644d2cada6dc44b363b7383dcd4fe., hostname=bd6592c37bb2,40087,1732486423713, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8ca644d2cada6dc44b363b7383dcd4fe, server=bd6592c37bb2,40087,1732486423713 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T22:16:40,838 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732486585339.8ca644d2cada6dc44b363b7383dcd4fe., hostname=bd6592c37bb2,40087,1732486423713, seqNum=5 because the exception is null or not the one we care about 2024-11-24T22:16:40,842 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40087 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8ca644d2cada6dc44b363b7383dcd4fe, server=bd6592c37bb2,40087,1732486423713 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-24T22:16:40,842 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40087 {}] ipc.CallRunner(138): callId: 54 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48986 deadline: 1732486660838, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8ca644d2cada6dc44b363b7383dcd4fe, server=bd6592c37bb2,40087,1732486423713 2024-11-24T22:16:40,842 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732486585339.8ca644d2cada6dc44b363b7383dcd4fe., hostname=bd6592c37bb2,40087,1732486423713, seqNum=5 , the old value is region=TestAcidGuarantees,,1732486585339.8ca644d2cada6dc44b363b7383dcd4fe., hostname=bd6592c37bb2,40087,1732486423713, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8ca644d2cada6dc44b363b7383dcd4fe, server=bd6592c37bb2,40087,1732486423713 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T22:16:40,842 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732486585339.8ca644d2cada6dc44b363b7383dcd4fe., hostname=bd6592c37bb2,40087,1732486423713, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8ca644d2cada6dc44b363b7383dcd4fe, server=bd6592c37bb2,40087,1732486423713 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T22:16:40,842 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732486585339.8ca644d2cada6dc44b363b7383dcd4fe., hostname=bd6592c37bb2,40087,1732486423713, seqNum=5 because the exception is null or not the one we care about 2024-11-24T22:16:40,943 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40087 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8ca644d2cada6dc44b363b7383dcd4fe, server=bd6592c37bb2,40087,1732486423713 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-24T22:16:40,943 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40087 {}] ipc.CallRunner(138): callId: 60 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49050 deadline: 1732486660940, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8ca644d2cada6dc44b363b7383dcd4fe, server=bd6592c37bb2,40087,1732486423713 2024-11-24T22:16:40,943 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40087 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8ca644d2cada6dc44b363b7383dcd4fe, server=bd6592c37bb2,40087,1732486423713 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-24T22:16:40,943 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40087 {}] ipc.CallRunner(138): callId: 53 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49002 deadline: 1732486660941, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8ca644d2cada6dc44b363b7383dcd4fe, server=bd6592c37bb2,40087,1732486423713 2024-11-24T22:16:40,943 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732486585339.8ca644d2cada6dc44b363b7383dcd4fe., hostname=bd6592c37bb2,40087,1732486423713, seqNum=5 , the old value is region=TestAcidGuarantees,,1732486585339.8ca644d2cada6dc44b363b7383dcd4fe., hostname=bd6592c37bb2,40087,1732486423713, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8ca644d2cada6dc44b363b7383dcd4fe, server=bd6592c37bb2,40087,1732486423713 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T22:16:40,943 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732486585339.8ca644d2cada6dc44b363b7383dcd4fe., hostname=bd6592c37bb2,40087,1732486423713, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8ca644d2cada6dc44b363b7383dcd4fe, server=bd6592c37bb2,40087,1732486423713 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T22:16:40,943 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732486585339.8ca644d2cada6dc44b363b7383dcd4fe., hostname=bd6592c37bb2,40087,1732486423713, seqNum=5 because the exception is null or not the one we care about 2024-11-24T22:16:40,943 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732486585339.8ca644d2cada6dc44b363b7383dcd4fe., hostname=bd6592c37bb2,40087,1732486423713, seqNum=5 , the old value is region=TestAcidGuarantees,,1732486585339.8ca644d2cada6dc44b363b7383dcd4fe., hostname=bd6592c37bb2,40087,1732486423713, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8ca644d2cada6dc44b363b7383dcd4fe, server=bd6592c37bb2,40087,1732486423713 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T22:16:40,943 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732486585339.8ca644d2cada6dc44b363b7383dcd4fe., hostname=bd6592c37bb2,40087,1732486423713, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8ca644d2cada6dc44b363b7383dcd4fe, server=bd6592c37bb2,40087,1732486423713 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T22:16:40,943 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732486585339.8ca644d2cada6dc44b363b7383dcd4fe., hostname=bd6592c37bb2,40087,1732486423713, seqNum=5 because the exception is null or not the one we care about 2024-11-24T22:16:40,953 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40087 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8ca644d2cada6dc44b363b7383dcd4fe, server=bd6592c37bb2,40087,1732486423713 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-24T22:16:40,953 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40087 {}] ipc.CallRunner(138): callId: 55 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48986 deadline: 1732486660951, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8ca644d2cada6dc44b363b7383dcd4fe, server=bd6592c37bb2,40087,1732486423713 2024-11-24T22:16:40,953 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732486585339.8ca644d2cada6dc44b363b7383dcd4fe., hostname=bd6592c37bb2,40087,1732486423713, seqNum=5 , the old value is region=TestAcidGuarantees,,1732486585339.8ca644d2cada6dc44b363b7383dcd4fe., hostname=bd6592c37bb2,40087,1732486423713, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8ca644d2cada6dc44b363b7383dcd4fe, server=bd6592c37bb2,40087,1732486423713 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T22:16:40,953 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732486585339.8ca644d2cada6dc44b363b7383dcd4fe., hostname=bd6592c37bb2,40087,1732486423713, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8ca644d2cada6dc44b363b7383dcd4fe, server=bd6592c37bb2,40087,1732486423713 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T22:16:40,953 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732486585339.8ca644d2cada6dc44b363b7383dcd4fe., hostname=bd6592c37bb2,40087,1732486423713, seqNum=5 because the exception is null or not the one we care about 2024-11-24T22:16:41,071 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37701 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=158 2024-11-24T22:16:41,071 INFO [RPCClient-NioEventLoopGroup-4-12 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-24T22:16:41,072 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37701 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-24T22:16:41,072 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37701 {}] procedure2.ProcedureExecutor(1139): Stored pid=160, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=160, table=TestAcidGuarantees 2024-11-24T22:16:41,073 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37701 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=160 2024-11-24T22:16:41,073 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=160, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=160, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-24T22:16:41,074 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=160, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=160, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-24T22:16:41,074 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=161, ppid=160, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-24T22:16:41,154 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40087 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8ca644d2cada6dc44b363b7383dcd4fe, server=bd6592c37bb2,40087,1732486423713 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-24T22:16:41,154 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40087 {}] ipc.CallRunner(138): callId: 61 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49050 deadline: 1732486661150, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8ca644d2cada6dc44b363b7383dcd4fe, server=bd6592c37bb2,40087,1732486423713 2024-11-24T22:16:41,154 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40087 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8ca644d2cada6dc44b363b7383dcd4fe, server=bd6592c37bb2,40087,1732486423713 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-24T22:16:41,154 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732486585339.8ca644d2cada6dc44b363b7383dcd4fe., hostname=bd6592c37bb2,40087,1732486423713, seqNum=5 , the old value is region=TestAcidGuarantees,,1732486585339.8ca644d2cada6dc44b363b7383dcd4fe., hostname=bd6592c37bb2,40087,1732486423713, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8ca644d2cada6dc44b363b7383dcd4fe, server=bd6592c37bb2,40087,1732486423713 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T22:16:41,154 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40087 {}] ipc.CallRunner(138): callId: 54 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49002 deadline: 1732486661151, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8ca644d2cada6dc44b363b7383dcd4fe, server=bd6592c37bb2,40087,1732486423713 2024-11-24T22:16:41,154 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732486585339.8ca644d2cada6dc44b363b7383dcd4fe., hostname=bd6592c37bb2,40087,1732486423713, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8ca644d2cada6dc44b363b7383dcd4fe, server=bd6592c37bb2,40087,1732486423713 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T22:16:41,154 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732486585339.8ca644d2cada6dc44b363b7383dcd4fe., hostname=bd6592c37bb2,40087,1732486423713, seqNum=5 because the exception is null or not the one we care about 2024-11-24T22:16:41,154 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732486585339.8ca644d2cada6dc44b363b7383dcd4fe., hostname=bd6592c37bb2,40087,1732486423713, seqNum=5 , the old value is region=TestAcidGuarantees,,1732486585339.8ca644d2cada6dc44b363b7383dcd4fe., hostname=bd6592c37bb2,40087,1732486423713, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8ca644d2cada6dc44b363b7383dcd4fe, server=bd6592c37bb2,40087,1732486423713 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T22:16:41,154 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732486585339.8ca644d2cada6dc44b363b7383dcd4fe., hostname=bd6592c37bb2,40087,1732486423713, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8ca644d2cada6dc44b363b7383dcd4fe, server=bd6592c37bb2,40087,1732486423713 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T22:16:41,154 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732486585339.8ca644d2cada6dc44b363b7383dcd4fe., hostname=bd6592c37bb2,40087,1732486423713, seqNum=5 because the exception is null or not the one we care about 2024-11-24T22:16:41,164 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40087 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8ca644d2cada6dc44b363b7383dcd4fe, server=bd6592c37bb2,40087,1732486423713 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-24T22:16:41,164 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40087 {}] ipc.CallRunner(138): callId: 56 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48986 deadline: 1732486661161, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8ca644d2cada6dc44b363b7383dcd4fe, server=bd6592c37bb2,40087,1732486423713 2024-11-24T22:16:41,165 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732486585339.8ca644d2cada6dc44b363b7383dcd4fe., hostname=bd6592c37bb2,40087,1732486423713, seqNum=5 , the old value is region=TestAcidGuarantees,,1732486585339.8ca644d2cada6dc44b363b7383dcd4fe., hostname=bd6592c37bb2,40087,1732486423713, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8ca644d2cada6dc44b363b7383dcd4fe, server=bd6592c37bb2,40087,1732486423713 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T22:16:41,165 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732486585339.8ca644d2cada6dc44b363b7383dcd4fe., hostname=bd6592c37bb2,40087,1732486423713, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8ca644d2cada6dc44b363b7383dcd4fe, server=bd6592c37bb2,40087,1732486423713 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T22:16:41,165 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732486585339.8ca644d2cada6dc44b363b7383dcd4fe., hostname=bd6592c37bb2,40087,1732486423713, seqNum=5 because the exception is null or not the one we care about 2024-11-24T22:16:41,181 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37701 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=160 2024-11-24T22:16:41,226 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=40087 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=161 2024-11-24T22:16:41,227 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bd6592c37bb2:0-2 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732486585339.8ca644d2cada6dc44b363b7383dcd4fe. 2024-11-24T22:16:41,227 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bd6592c37bb2:0-2 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732486585339.8ca644d2cada6dc44b363b7383dcd4fe. as already flushing 2024-11-24T22:16:41,227 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bd6592c37bb2:0-2 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732486585339.8ca644d2cada6dc44b363b7383dcd4fe. 2024-11-24T22:16:41,227 ERROR [RS_FLUSH_OPERATIONS-regionserver/bd6592c37bb2:0-2 {event_type=RS_FLUSH_REGIONS, pid=161}] handler.RSProcedureHandler(58): pid=161 java.io.IOException: Unable to complete flush {ENCODED => 8ca644d2cada6dc44b363b7383dcd4fe, NAME => 'TestAcidGuarantees,,1732486585339.8ca644d2cada6dc44b363b7383dcd4fe.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T22:16:41,227 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bd6592c37bb2:0-2 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=161 java.io.IOException: Unable to complete flush {ENCODED => 8ca644d2cada6dc44b363b7383dcd4fe, NAME => 'TestAcidGuarantees,,1732486585339.8ca644d2cada6dc44b363b7383dcd4fe.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T22:16:41,228 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37701 {}] master.HMaster(4177): Remote procedure failed, pid=161 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 8ca644d2cada6dc44b363b7383dcd4fe, NAME => 'TestAcidGuarantees,,1732486585339.8ca644d2cada6dc44b363b7383dcd4fe.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 8ca644d2cada6dc44b363b7383dcd4fe, NAME => 'TestAcidGuarantees,,1732486585339.8ca644d2cada6dc44b363b7383dcd4fe.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T22:16:41,229 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T22:16:41,232 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241124df325bb5e0354ee5be9c82bf6a0b1763_8ca644d2cada6dc44b363b7383dcd4fe to hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241124df325bb5e0354ee5be9c82bf6a0b1763_8ca644d2cada6dc44b363b7383dcd4fe 2024-11-24T22:16:41,233 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/8ca644d2cada6dc44b363b7383dcd4fe/.tmp/A/dd9c9897b4964ee9af1242e61e3c3d45, store: [table=TestAcidGuarantees family=A region=8ca644d2cada6dc44b363b7383dcd4fe] 2024-11-24T22:16:41,233 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/8ca644d2cada6dc44b363b7383dcd4fe/.tmp/A/dd9c9897b4964ee9af1242e61e3c3d45 is 175, key is test_row_0/A:col10/1732486600150/Put/seqid=0 2024-11-24T22:16:41,237 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45929 is added to blk_1073742431_1607 (size=48389) 2024-11-24T22:16:41,237 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=187, memsize=33.5 K, hasBloomFilter=true, into tmp file hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/8ca644d2cada6dc44b363b7383dcd4fe/.tmp/A/dd9c9897b4964ee9af1242e61e3c3d45 2024-11-24T22:16:41,243 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/8ca644d2cada6dc44b363b7383dcd4fe/.tmp/B/6e64a39e7a474ab9a6e384c6fbeb34d3 is 50, key is test_row_0/B:col10/1732486600150/Put/seqid=0 2024-11-24T22:16:41,246 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45929 is added to blk_1073742432_1608 (size=12151) 2024-11-24T22:16:41,247 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=33.54 KB at sequenceid=187 (bloomFilter=true), to=hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/8ca644d2cada6dc44b363b7383dcd4fe/.tmp/B/6e64a39e7a474ab9a6e384c6fbeb34d3 2024-11-24T22:16:41,253 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/8ca644d2cada6dc44b363b7383dcd4fe/.tmp/C/0757bc93a25a45b7a830589a9f99e3d8 is 50, key is test_row_0/C:col10/1732486600150/Put/seqid=0 2024-11-24T22:16:41,256 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45929 is added to blk_1073742433_1609 (size=12151) 2024-11-24T22:16:41,379 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=40087 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=161 2024-11-24T22:16:41,379 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bd6592c37bb2:0-0 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732486585339.8ca644d2cada6dc44b363b7383dcd4fe. 2024-11-24T22:16:41,380 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bd6592c37bb2:0-0 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732486585339.8ca644d2cada6dc44b363b7383dcd4fe. as already flushing 2024-11-24T22:16:41,380 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bd6592c37bb2:0-0 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732486585339.8ca644d2cada6dc44b363b7383dcd4fe. 2024-11-24T22:16:41,380 ERROR [RS_FLUSH_OPERATIONS-regionserver/bd6592c37bb2:0-0 {event_type=RS_FLUSH_REGIONS, pid=161}] handler.RSProcedureHandler(58): pid=161 java.io.IOException: Unable to complete flush {ENCODED => 8ca644d2cada6dc44b363b7383dcd4fe, NAME => 'TestAcidGuarantees,,1732486585339.8ca644d2cada6dc44b363b7383dcd4fe.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T22:16:41,380 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bd6592c37bb2:0-0 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=161 java.io.IOException: Unable to complete flush {ENCODED => 8ca644d2cada6dc44b363b7383dcd4fe, NAME => 'TestAcidGuarantees,,1732486585339.8ca644d2cada6dc44b363b7383dcd4fe.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T22:16:41,381 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37701 {}] master.HMaster(4177): Remote procedure failed, pid=161 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 8ca644d2cada6dc44b363b7383dcd4fe, NAME => 'TestAcidGuarantees,,1732486585339.8ca644d2cada6dc44b363b7383dcd4fe.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 8ca644d2cada6dc44b363b7383dcd4fe, NAME => 'TestAcidGuarantees,,1732486585339.8ca644d2cada6dc44b363b7383dcd4fe.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T22:16:41,391 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37701 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=160 2024-11-24T22:16:41,462 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40087 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8ca644d2cada6dc44b363b7383dcd4fe, server=bd6592c37bb2,40087,1732486423713 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-24T22:16:41,462 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40087 {}] ipc.CallRunner(138): callId: 62 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49050 deadline: 1732486661460, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8ca644d2cada6dc44b363b7383dcd4fe, server=bd6592c37bb2,40087,1732486423713 2024-11-24T22:16:41,462 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40087 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8ca644d2cada6dc44b363b7383dcd4fe, server=bd6592c37bb2,40087,1732486423713 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-24T22:16:41,463 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40087 {}] ipc.CallRunner(138): callId: 55 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49002 deadline: 1732486661460, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8ca644d2cada6dc44b363b7383dcd4fe, server=bd6592c37bb2,40087,1732486423713 2024-11-24T22:16:41,463 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732486585339.8ca644d2cada6dc44b363b7383dcd4fe., hostname=bd6592c37bb2,40087,1732486423713, seqNum=5 , the old value is region=TestAcidGuarantees,,1732486585339.8ca644d2cada6dc44b363b7383dcd4fe., hostname=bd6592c37bb2,40087,1732486423713, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8ca644d2cada6dc44b363b7383dcd4fe, server=bd6592c37bb2,40087,1732486423713 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T22:16:41,463 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732486585339.8ca644d2cada6dc44b363b7383dcd4fe., hostname=bd6592c37bb2,40087,1732486423713, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8ca644d2cada6dc44b363b7383dcd4fe, server=bd6592c37bb2,40087,1732486423713 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T22:16:41,463 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732486585339.8ca644d2cada6dc44b363b7383dcd4fe., hostname=bd6592c37bb2,40087,1732486423713, seqNum=5 because the exception is null or not the one we care about 2024-11-24T22:16:41,463 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732486585339.8ca644d2cada6dc44b363b7383dcd4fe., hostname=bd6592c37bb2,40087,1732486423713, seqNum=5 , the old value is region=TestAcidGuarantees,,1732486585339.8ca644d2cada6dc44b363b7383dcd4fe., hostname=bd6592c37bb2,40087,1732486423713, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8ca644d2cada6dc44b363b7383dcd4fe, server=bd6592c37bb2,40087,1732486423713 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T22:16:41,463 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732486585339.8ca644d2cada6dc44b363b7383dcd4fe., hostname=bd6592c37bb2,40087,1732486423713, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8ca644d2cada6dc44b363b7383dcd4fe, server=bd6592c37bb2,40087,1732486423713 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T22:16:41,463 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732486585339.8ca644d2cada6dc44b363b7383dcd4fe., hostname=bd6592c37bb2,40087,1732486423713, seqNum=5 because the exception is null or not the one we care about 2024-11-24T22:16:41,472 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40087 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8ca644d2cada6dc44b363b7383dcd4fe, server=bd6592c37bb2,40087,1732486423713 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-24T22:16:41,473 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40087 {}] ipc.CallRunner(138): callId: 57 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48986 deadline: 1732486661471, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8ca644d2cada6dc44b363b7383dcd4fe, server=bd6592c37bb2,40087,1732486423713 2024-11-24T22:16:41,473 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732486585339.8ca644d2cada6dc44b363b7383dcd4fe., hostname=bd6592c37bb2,40087,1732486423713, seqNum=5 , the old value is region=TestAcidGuarantees,,1732486585339.8ca644d2cada6dc44b363b7383dcd4fe., hostname=bd6592c37bb2,40087,1732486423713, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8ca644d2cada6dc44b363b7383dcd4fe, server=bd6592c37bb2,40087,1732486423713 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T22:16:41,473 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732486585339.8ca644d2cada6dc44b363b7383dcd4fe., hostname=bd6592c37bb2,40087,1732486423713, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8ca644d2cada6dc44b363b7383dcd4fe, server=bd6592c37bb2,40087,1732486423713 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T22:16:41,473 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732486585339.8ca644d2cada6dc44b363b7383dcd4fe., hostname=bd6592c37bb2,40087,1732486423713, seqNum=5 because the exception is null or not the one we care about 2024-11-24T22:16:41,532 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=40087 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=161 2024-11-24T22:16:41,533 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bd6592c37bb2:0-1 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732486585339.8ca644d2cada6dc44b363b7383dcd4fe. 2024-11-24T22:16:41,533 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bd6592c37bb2:0-1 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732486585339.8ca644d2cada6dc44b363b7383dcd4fe. as already flushing 2024-11-24T22:16:41,533 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bd6592c37bb2:0-1 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732486585339.8ca644d2cada6dc44b363b7383dcd4fe. 2024-11-24T22:16:41,533 ERROR [RS_FLUSH_OPERATIONS-regionserver/bd6592c37bb2:0-1 {event_type=RS_FLUSH_REGIONS, pid=161}] handler.RSProcedureHandler(58): pid=161 java.io.IOException: Unable to complete flush {ENCODED => 8ca644d2cada6dc44b363b7383dcd4fe, NAME => 'TestAcidGuarantees,,1732486585339.8ca644d2cada6dc44b363b7383dcd4fe.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T22:16:41,533 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bd6592c37bb2:0-1 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=161 java.io.IOException: Unable to complete flush {ENCODED => 8ca644d2cada6dc44b363b7383dcd4fe, NAME => 'TestAcidGuarantees,,1732486585339.8ca644d2cada6dc44b363b7383dcd4fe.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T22:16:41,533 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37701 {}] master.HMaster(4177): Remote procedure failed, pid=161 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 8ca644d2cada6dc44b363b7383dcd4fe, NAME => 'TestAcidGuarantees,,1732486585339.8ca644d2cada6dc44b363b7383dcd4fe.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 8ca644d2cada6dc44b363b7383dcd4fe, NAME => 'TestAcidGuarantees,,1732486585339.8ca644d2cada6dc44b363b7383dcd4fe.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T22:16:41,657 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=33.54 KB at sequenceid=187 (bloomFilter=true), to=hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/8ca644d2cada6dc44b363b7383dcd4fe/.tmp/C/0757bc93a25a45b7a830589a9f99e3d8 2024-11-24T22:16:41,660 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/8ca644d2cada6dc44b363b7383dcd4fe/.tmp/A/dd9c9897b4964ee9af1242e61e3c3d45 as hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/8ca644d2cada6dc44b363b7383dcd4fe/A/dd9c9897b4964ee9af1242e61e3c3d45 2024-11-24T22:16:41,663 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/8ca644d2cada6dc44b363b7383dcd4fe/A/dd9c9897b4964ee9af1242e61e3c3d45, entries=250, sequenceid=187, filesize=47.3 K 2024-11-24T22:16:41,664 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/8ca644d2cada6dc44b363b7383dcd4fe/.tmp/B/6e64a39e7a474ab9a6e384c6fbeb34d3 as hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/8ca644d2cada6dc44b363b7383dcd4fe/B/6e64a39e7a474ab9a6e384c6fbeb34d3 2024-11-24T22:16:41,667 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/8ca644d2cada6dc44b363b7383dcd4fe/B/6e64a39e7a474ab9a6e384c6fbeb34d3, entries=150, sequenceid=187, filesize=11.9 K 2024-11-24T22:16:41,667 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/8ca644d2cada6dc44b363b7383dcd4fe/.tmp/C/0757bc93a25a45b7a830589a9f99e3d8 as hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/8ca644d2cada6dc44b363b7383dcd4fe/C/0757bc93a25a45b7a830589a9f99e3d8 2024-11-24T22:16:41,670 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/8ca644d2cada6dc44b363b7383dcd4fe/C/0757bc93a25a45b7a830589a9f99e3d8, entries=150, sequenceid=187, filesize=11.9 K 2024-11-24T22:16:41,671 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~100.63 KB/103050, heapSize ~299.53 KB/306720, currentSize=73.80 KB/75570 for 8ca644d2cada6dc44b363b7383dcd4fe in 858ms, sequenceid=187, compaction requested=true 2024-11-24T22:16:41,671 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 8ca644d2cada6dc44b363b7383dcd4fe: 2024-11-24T22:16:41,671 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 8ca644d2cada6dc44b363b7383dcd4fe:A, priority=-2147483648, current under compaction store size is 1 2024-11-24T22:16:41,671 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-24T22:16:41,671 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 8ca644d2cada6dc44b363b7383dcd4fe:B, priority=-2147483648, current under compaction store size is 2 2024-11-24T22:16:41,671 DEBUG [RS:0;bd6592c37bb2:40087-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-24T22:16:41,671 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-24T22:16:41,671 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 8ca644d2cada6dc44b363b7383dcd4fe:C, priority=-2147483648, current under compaction store size is 3 2024-11-24T22:16:41,671 DEBUG [RS:0;bd6592c37bb2:40087-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-24T22:16:41,671 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-24T22:16:41,672 DEBUG [RS:0;bd6592c37bb2:40087-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 110907 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-24T22:16:41,672 DEBUG [RS:0;bd6592c37bb2:40087-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36761 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-24T22:16:41,672 DEBUG [RS:0;bd6592c37bb2:40087-shortCompactions-0 {}] regionserver.HStore(1541): 8ca644d2cada6dc44b363b7383dcd4fe/A is initiating minor compaction (all files) 2024-11-24T22:16:41,672 DEBUG [RS:0;bd6592c37bb2:40087-longCompactions-0 {}] regionserver.HStore(1541): 8ca644d2cada6dc44b363b7383dcd4fe/B is initiating minor compaction (all files) 2024-11-24T22:16:41,672 INFO [RS:0;bd6592c37bb2:40087-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 8ca644d2cada6dc44b363b7383dcd4fe/B in TestAcidGuarantees,,1732486585339.8ca644d2cada6dc44b363b7383dcd4fe. 2024-11-24T22:16:41,672 INFO [RS:0;bd6592c37bb2:40087-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 8ca644d2cada6dc44b363b7383dcd4fe/A in TestAcidGuarantees,,1732486585339.8ca644d2cada6dc44b363b7383dcd4fe. 2024-11-24T22:16:41,672 INFO [RS:0;bd6592c37bb2:40087-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/8ca644d2cada6dc44b363b7383dcd4fe/B/5037fa9e4f23489aa5eede59338d5122, hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/8ca644d2cada6dc44b363b7383dcd4fe/B/31584a242fc64dfcb888a07f8cbddd9f, hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/8ca644d2cada6dc44b363b7383dcd4fe/B/6e64a39e7a474ab9a6e384c6fbeb34d3] into tmpdir=hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/8ca644d2cada6dc44b363b7383dcd4fe/.tmp, totalSize=35.9 K 2024-11-24T22:16:41,672 INFO [RS:0;bd6592c37bb2:40087-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/8ca644d2cada6dc44b363b7383dcd4fe/A/b722343fe5fc48559de2c4b094712e51, hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/8ca644d2cada6dc44b363b7383dcd4fe/A/c679a2427e9f4fcea97a02498a755dde, hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/8ca644d2cada6dc44b363b7383dcd4fe/A/dd9c9897b4964ee9af1242e61e3c3d45] into tmpdir=hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/8ca644d2cada6dc44b363b7383dcd4fe/.tmp, totalSize=108.3 K 2024-11-24T22:16:41,672 INFO [RS:0;bd6592c37bb2:40087-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=13 throughput controller=DefaultCompactionThroughputController [maxThroughput=57.69 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1732486585339.8ca644d2cada6dc44b363b7383dcd4fe. 2024-11-24T22:16:41,672 DEBUG [RS:0;bd6592c37bb2:40087-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1732486585339.8ca644d2cada6dc44b363b7383dcd4fe. files: [hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/8ca644d2cada6dc44b363b7383dcd4fe/A/b722343fe5fc48559de2c4b094712e51, hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/8ca644d2cada6dc44b363b7383dcd4fe/A/c679a2427e9f4fcea97a02498a755dde, hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/8ca644d2cada6dc44b363b7383dcd4fe/A/dd9c9897b4964ee9af1242e61e3c3d45] 2024-11-24T22:16:41,672 DEBUG [RS:0;bd6592c37bb2:40087-longCompactions-0 {}] compactions.Compactor(225): Compacting 5037fa9e4f23489aa5eede59338d5122, keycount=150, bloomtype=ROW, size=12.2 K, encoding=NONE, compression=NONE, seqNum=149, earliestPutTs=1732486596773 2024-11-24T22:16:41,672 DEBUG [RS:0;bd6592c37bb2:40087-shortCompactions-0 {}] compactions.Compactor(225): Compacting b722343fe5fc48559de2c4b094712e51, keycount=150, bloomtype=ROW, size=30.7 K, encoding=NONE, compression=NONE, seqNum=149, earliestPutTs=1732486596773 2024-11-24T22:16:41,673 DEBUG [RS:0;bd6592c37bb2:40087-longCompactions-0 {}] compactions.Compactor(225): Compacting 31584a242fc64dfcb888a07f8cbddd9f, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=166, earliestPutTs=1732486597988 2024-11-24T22:16:41,673 DEBUG [RS:0;bd6592c37bb2:40087-shortCompactions-0 {}] compactions.Compactor(225): Compacting c679a2427e9f4fcea97a02498a755dde, keycount=150, bloomtype=ROW, size=30.4 K, encoding=NONE, compression=NONE, seqNum=166, earliestPutTs=1732486597988 2024-11-24T22:16:41,673 DEBUG [RS:0;bd6592c37bb2:40087-longCompactions-0 {}] compactions.Compactor(225): Compacting 6e64a39e7a474ab9a6e384c6fbeb34d3, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=187, earliestPutTs=1732486600150 2024-11-24T22:16:41,673 DEBUG [RS:0;bd6592c37bb2:40087-shortCompactions-0 {}] compactions.Compactor(225): Compacting dd9c9897b4964ee9af1242e61e3c3d45, keycount=250, bloomtype=ROW, size=47.3 K, encoding=NONE, compression=NONE, seqNum=187, earliestPutTs=1732486600150 2024-11-24T22:16:41,678 INFO [RS:0;bd6592c37bb2:40087-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=8ca644d2cada6dc44b363b7383dcd4fe] 2024-11-24T22:16:41,679 INFO [RS:0;bd6592c37bb2:40087-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 8ca644d2cada6dc44b363b7383dcd4fe#B#compaction#515 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 57.69 MB/second 2024-11-24T22:16:41,679 DEBUG [RS:0;bd6592c37bb2:40087-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e202411241cf6900b585040a39c3e82bcd09c72b0_8ca644d2cada6dc44b363b7383dcd4fe store=[table=TestAcidGuarantees family=A region=8ca644d2cada6dc44b363b7383dcd4fe] 2024-11-24T22:16:41,680 DEBUG [RS:0;bd6592c37bb2:40087-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/8ca644d2cada6dc44b363b7383dcd4fe/.tmp/B/730efb19372148f8bdb95e42b451eb85 is 50, key is test_row_0/B:col10/1732486600150/Put/seqid=0 2024-11-24T22:16:41,682 DEBUG [RS:0;bd6592c37bb2:40087-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e202411241cf6900b585040a39c3e82bcd09c72b0_8ca644d2cada6dc44b363b7383dcd4fe, store=[table=TestAcidGuarantees family=A region=8ca644d2cada6dc44b363b7383dcd4fe] 2024-11-24T22:16:41,682 DEBUG [RS:0;bd6592c37bb2:40087-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411241cf6900b585040a39c3e82bcd09c72b0_8ca644d2cada6dc44b363b7383dcd4fe because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=8ca644d2cada6dc44b363b7383dcd4fe] 2024-11-24T22:16:41,685 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=40087 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=161 2024-11-24T22:16:41,685 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bd6592c37bb2:0-2 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732486585339.8ca644d2cada6dc44b363b7383dcd4fe. 2024-11-24T22:16:41,686 INFO [RS_FLUSH_OPERATIONS-regionserver/bd6592c37bb2:0-2 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.HRegion(2902): Flushing 8ca644d2cada6dc44b363b7383dcd4fe 3/3 column families, dataSize=73.80 KB heapSize=219.89 KB 2024-11-24T22:16:41,686 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bd6592c37bb2:0-2 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 8ca644d2cada6dc44b363b7383dcd4fe, store=A 2024-11-24T22:16:41,686 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bd6592c37bb2:0-2 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-24T22:16:41,686 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bd6592c37bb2:0-2 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 8ca644d2cada6dc44b363b7383dcd4fe, store=B 2024-11-24T22:16:41,686 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bd6592c37bb2:0-2 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-24T22:16:41,686 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bd6592c37bb2:0-2 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 8ca644d2cada6dc44b363b7383dcd4fe, store=C 2024-11-24T22:16:41,686 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bd6592c37bb2:0-2 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-24T22:16:41,699 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45929 is added to blk_1073742434_1610 (size=4469) 2024-11-24T22:16:41,700 INFO [RS:0;bd6592c37bb2:40087-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 8ca644d2cada6dc44b363b7383dcd4fe#A#compaction#516 average throughput is 1.11 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 57.69 MB/second 2024-11-24T22:16:41,700 DEBUG [RS:0;bd6592c37bb2:40087-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/8ca644d2cada6dc44b363b7383dcd4fe/.tmp/A/09f6f98a4dde46df9e1923bd989b27de is 175, key is test_row_0/A:col10/1732486600150/Put/seqid=0 2024-11-24T22:16:41,701 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37701 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=160 2024-11-24T22:16:41,710 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bd6592c37bb2:0-2 {event_type=RS_FLUSH_REGIONS, pid=161}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241124228b01d3b9244ae0892af72a99fd9e63_8ca644d2cada6dc44b363b7383dcd4fe is 50, key is test_row_0/A:col10/1732486600836/Put/seqid=0 2024-11-24T22:16:41,710 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45929 is added to blk_1073742435_1611 (size=12561) 2024-11-24T22:16:41,725 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45929 is added to blk_1073742437_1613 (size=12304) 2024-11-24T22:16:41,726 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bd6592c37bb2:0-2 {event_type=RS_FLUSH_REGIONS, pid=161}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T22:16:41,730 INFO [RS_FLUSH_OPERATIONS-regionserver/bd6592c37bb2:0-2 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241124228b01d3b9244ae0892af72a99fd9e63_8ca644d2cada6dc44b363b7383dcd4fe to hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241124228b01d3b9244ae0892af72a99fd9e63_8ca644d2cada6dc44b363b7383dcd4fe 2024-11-24T22:16:41,731 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bd6592c37bb2:0-2 {event_type=RS_FLUSH_REGIONS, pid=161}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/8ca644d2cada6dc44b363b7383dcd4fe/.tmp/A/b8d8d281b8a04eeb8945f0979f39bfcc, store: [table=TestAcidGuarantees family=A region=8ca644d2cada6dc44b363b7383dcd4fe] 2024-11-24T22:16:41,731 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bd6592c37bb2:0-2 {event_type=RS_FLUSH_REGIONS, pid=161}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/8ca644d2cada6dc44b363b7383dcd4fe/.tmp/A/b8d8d281b8a04eeb8945f0979f39bfcc is 175, key is test_row_0/A:col10/1732486600836/Put/seqid=0 2024-11-24T22:16:41,738 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45929 is added to blk_1073742436_1612 (size=31515) 2024-11-24T22:16:41,739 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45929 is added to blk_1073742438_1614 (size=31105) 2024-11-24T22:16:41,740 INFO [RS_FLUSH_OPERATIONS-regionserver/bd6592c37bb2:0-2 {event_type=RS_FLUSH_REGIONS, pid=161}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=201, memsize=24.6 K, hasBloomFilter=true, into tmp file hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/8ca644d2cada6dc44b363b7383dcd4fe/.tmp/A/b8d8d281b8a04eeb8945f0979f39bfcc 2024-11-24T22:16:41,744 DEBUG [RS:0;bd6592c37bb2:40087-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/8ca644d2cada6dc44b363b7383dcd4fe/.tmp/A/09f6f98a4dde46df9e1923bd989b27de as hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/8ca644d2cada6dc44b363b7383dcd4fe/A/09f6f98a4dde46df9e1923bd989b27de 2024-11-24T22:16:41,748 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bd6592c37bb2:0-2 {event_type=RS_FLUSH_REGIONS, pid=161}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/8ca644d2cada6dc44b363b7383dcd4fe/.tmp/B/0a9cb8108ab84a25a32039517a52cbeb is 50, key is test_row_0/B:col10/1732486600836/Put/seqid=0 2024-11-24T22:16:41,748 INFO [RS:0;bd6592c37bb2:40087-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 8ca644d2cada6dc44b363b7383dcd4fe/A of 8ca644d2cada6dc44b363b7383dcd4fe into 09f6f98a4dde46df9e1923bd989b27de(size=30.8 K), total size for store is 30.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-24T22:16:41,748 DEBUG [RS:0;bd6592c37bb2:40087-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 8ca644d2cada6dc44b363b7383dcd4fe: 2024-11-24T22:16:41,748 INFO [RS:0;bd6592c37bb2:40087-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732486585339.8ca644d2cada6dc44b363b7383dcd4fe., storeName=8ca644d2cada6dc44b363b7383dcd4fe/A, priority=13, startTime=1732486601671; duration=0sec 2024-11-24T22:16:41,748 DEBUG [RS:0;bd6592c37bb2:40087-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-24T22:16:41,748 DEBUG [RS:0;bd6592c37bb2:40087-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 8ca644d2cada6dc44b363b7383dcd4fe:A 2024-11-24T22:16:41,748 DEBUG [RS:0;bd6592c37bb2:40087-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-24T22:16:41,750 DEBUG [RS:0;bd6592c37bb2:40087-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36761 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-24T22:16:41,750 DEBUG [RS:0;bd6592c37bb2:40087-shortCompactions-0 {}] regionserver.HStore(1541): 8ca644d2cada6dc44b363b7383dcd4fe/C is initiating minor compaction (all files) 2024-11-24T22:16:41,750 INFO [RS:0;bd6592c37bb2:40087-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 8ca644d2cada6dc44b363b7383dcd4fe/C in TestAcidGuarantees,,1732486585339.8ca644d2cada6dc44b363b7383dcd4fe. 2024-11-24T22:16:41,750 INFO [RS:0;bd6592c37bb2:40087-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/8ca644d2cada6dc44b363b7383dcd4fe/C/4c146cfb566b43de8d7bda68c062baa8, hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/8ca644d2cada6dc44b363b7383dcd4fe/C/0fec38d8c31a41a8a5233d65261f9229, hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/8ca644d2cada6dc44b363b7383dcd4fe/C/0757bc93a25a45b7a830589a9f99e3d8] into tmpdir=hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/8ca644d2cada6dc44b363b7383dcd4fe/.tmp, totalSize=35.9 K 2024-11-24T22:16:41,751 DEBUG [RS:0;bd6592c37bb2:40087-shortCompactions-0 {}] compactions.Compactor(225): Compacting 4c146cfb566b43de8d7bda68c062baa8, keycount=150, bloomtype=ROW, size=12.2 K, encoding=NONE, compression=NONE, seqNum=149, earliestPutTs=1732486596773 2024-11-24T22:16:41,751 DEBUG [RS:0;bd6592c37bb2:40087-shortCompactions-0 {}] compactions.Compactor(225): Compacting 0fec38d8c31a41a8a5233d65261f9229, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=166, earliestPutTs=1732486597988 2024-11-24T22:16:41,751 DEBUG [RS:0;bd6592c37bb2:40087-shortCompactions-0 {}] compactions.Compactor(225): Compacting 0757bc93a25a45b7a830589a9f99e3d8, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=187, earliestPutTs=1732486600150 2024-11-24T22:16:41,752 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45929 is added to blk_1073742439_1615 (size=12151) 2024-11-24T22:16:41,753 INFO [RS_FLUSH_OPERATIONS-regionserver/bd6592c37bb2:0-2 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=24.60 KB at sequenceid=201 (bloomFilter=true), to=hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/8ca644d2cada6dc44b363b7383dcd4fe/.tmp/B/0a9cb8108ab84a25a32039517a52cbeb 2024-11-24T22:16:41,759 INFO [RS:0;bd6592c37bb2:40087-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 8ca644d2cada6dc44b363b7383dcd4fe#C#compaction#519 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 57.69 MB/second 2024-11-24T22:16:41,759 DEBUG [RS:0;bd6592c37bb2:40087-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/8ca644d2cada6dc44b363b7383dcd4fe/.tmp/C/09a368f67227449abf941025f6c40c9f is 50, key is test_row_0/C:col10/1732486600150/Put/seqid=0 2024-11-24T22:16:41,763 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45929 is added to blk_1073742440_1616 (size=12561) 2024-11-24T22:16:41,765 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bd6592c37bb2:0-2 {event_type=RS_FLUSH_REGIONS, pid=161}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/8ca644d2cada6dc44b363b7383dcd4fe/.tmp/C/b6da2cc25fa44492ad282381b9a45c04 is 50, key is test_row_0/C:col10/1732486600836/Put/seqid=0 2024-11-24T22:16:41,769 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45929 is added to blk_1073742441_1617 (size=12151) 2024-11-24T22:16:41,973 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732486585339.8ca644d2cada6dc44b363b7383dcd4fe. as already flushing 2024-11-24T22:16:41,973 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40087 {}] regionserver.HRegion(8855): Flush requested on 8ca644d2cada6dc44b363b7383dcd4fe 2024-11-24T22:16:42,011 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40087 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8ca644d2cada6dc44b363b7383dcd4fe, server=bd6592c37bb2,40087,1732486423713 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-24T22:16:42,011 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40087 {}] ipc.CallRunner(138): callId: 69 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49050 deadline: 1732486662004, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8ca644d2cada6dc44b363b7383dcd4fe, server=bd6592c37bb2,40087,1732486423713 2024-11-24T22:16:42,011 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40087 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8ca644d2cada6dc44b363b7383dcd4fe, server=bd6592c37bb2,40087,1732486423713 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-24T22:16:42,011 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40087 {}] ipc.CallRunner(138): callId: 62 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49002 deadline: 1732486662004, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8ca644d2cada6dc44b363b7383dcd4fe, server=bd6592c37bb2,40087,1732486423713 2024-11-24T22:16:42,011 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732486585339.8ca644d2cada6dc44b363b7383dcd4fe., hostname=bd6592c37bb2,40087,1732486423713, seqNum=5 , the old value is region=TestAcidGuarantees,,1732486585339.8ca644d2cada6dc44b363b7383dcd4fe., hostname=bd6592c37bb2,40087,1732486423713, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8ca644d2cada6dc44b363b7383dcd4fe, server=bd6592c37bb2,40087,1732486423713 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T22:16:42,011 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732486585339.8ca644d2cada6dc44b363b7383dcd4fe., hostname=bd6592c37bb2,40087,1732486423713, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8ca644d2cada6dc44b363b7383dcd4fe, server=bd6592c37bb2,40087,1732486423713 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T22:16:42,012 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732486585339.8ca644d2cada6dc44b363b7383dcd4fe., hostname=bd6592c37bb2,40087,1732486423713, seqNum=5 because the exception is null or not the one we care about 2024-11-24T22:16:42,012 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40087 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8ca644d2cada6dc44b363b7383dcd4fe, server=bd6592c37bb2,40087,1732486423713 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-24T22:16:42,012 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732486585339.8ca644d2cada6dc44b363b7383dcd4fe., hostname=bd6592c37bb2,40087,1732486423713, seqNum=5 , the old value is region=TestAcidGuarantees,,1732486585339.8ca644d2cada6dc44b363b7383dcd4fe., hostname=bd6592c37bb2,40087,1732486423713, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8ca644d2cada6dc44b363b7383dcd4fe, server=bd6592c37bb2,40087,1732486423713 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T22:16:42,012 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40087 {}] ipc.CallRunner(138): callId: 62 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48986 deadline: 1732486662005, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8ca644d2cada6dc44b363b7383dcd4fe, server=bd6592c37bb2,40087,1732486423713 2024-11-24T22:16:42,012 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732486585339.8ca644d2cada6dc44b363b7383dcd4fe., hostname=bd6592c37bb2,40087,1732486423713, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8ca644d2cada6dc44b363b7383dcd4fe, server=bd6592c37bb2,40087,1732486423713 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T22:16:42,012 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732486585339.8ca644d2cada6dc44b363b7383dcd4fe., hostname=bd6592c37bb2,40087,1732486423713, seqNum=5 because the exception is null or not the one we care about 2024-11-24T22:16:42,012 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732486585339.8ca644d2cada6dc44b363b7383dcd4fe., hostname=bd6592c37bb2,40087,1732486423713, seqNum=5 , the old value is region=TestAcidGuarantees,,1732486585339.8ca644d2cada6dc44b363b7383dcd4fe., hostname=bd6592c37bb2,40087,1732486423713, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8ca644d2cada6dc44b363b7383dcd4fe, server=bd6592c37bb2,40087,1732486423713 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T22:16:42,012 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732486585339.8ca644d2cada6dc44b363b7383dcd4fe., hostname=bd6592c37bb2,40087,1732486423713, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8ca644d2cada6dc44b363b7383dcd4fe, server=bd6592c37bb2,40087,1732486423713 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T22:16:42,012 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732486585339.8ca644d2cada6dc44b363b7383dcd4fe., hostname=bd6592c37bb2,40087,1732486423713, seqNum=5 because the exception is null or not the one we care about 2024-11-24T22:16:42,016 DEBUG [FsDatasetAsyncDiskServiceFixer {}] hbase.HBaseTestingUtil$FsDatasetAsyncDiskServiceFixer(597): NoSuchFieldException: threadGroup; It might because your Hadoop version > 3.2.3 or 3.3.4, See HBASE-27595 for details. 2024-11-24T22:16:42,115 DEBUG [RS:0;bd6592c37bb2:40087-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/8ca644d2cada6dc44b363b7383dcd4fe/.tmp/B/730efb19372148f8bdb95e42b451eb85 as hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/8ca644d2cada6dc44b363b7383dcd4fe/B/730efb19372148f8bdb95e42b451eb85 2024-11-24T22:16:42,118 INFO [RS:0;bd6592c37bb2:40087-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 8ca644d2cada6dc44b363b7383dcd4fe/B of 8ca644d2cada6dc44b363b7383dcd4fe into 730efb19372148f8bdb95e42b451eb85(size=12.3 K), total size for store is 12.3 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-24T22:16:42,118 DEBUG [RS:0;bd6592c37bb2:40087-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 8ca644d2cada6dc44b363b7383dcd4fe: 2024-11-24T22:16:42,118 INFO [RS:0;bd6592c37bb2:40087-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732486585339.8ca644d2cada6dc44b363b7383dcd4fe., storeName=8ca644d2cada6dc44b363b7383dcd4fe/B, priority=13, startTime=1732486601671; duration=0sec 2024-11-24T22:16:42,118 DEBUG [RS:0;bd6592c37bb2:40087-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-24T22:16:42,118 DEBUG [RS:0;bd6592c37bb2:40087-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 8ca644d2cada6dc44b363b7383dcd4fe:B 2024-11-24T22:16:42,124 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40087 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8ca644d2cada6dc44b363b7383dcd4fe, server=bd6592c37bb2,40087,1732486423713 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-24T22:16:42,124 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40087 {}] ipc.CallRunner(138): callId: 70 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49050 deadline: 1732486662121, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8ca644d2cada6dc44b363b7383dcd4fe, server=bd6592c37bb2,40087,1732486423713 2024-11-24T22:16:42,124 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40087 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8ca644d2cada6dc44b363b7383dcd4fe, server=bd6592c37bb2,40087,1732486423713 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-24T22:16:42,124 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732486585339.8ca644d2cada6dc44b363b7383dcd4fe., hostname=bd6592c37bb2,40087,1732486423713, seqNum=5 , the old value is region=TestAcidGuarantees,,1732486585339.8ca644d2cada6dc44b363b7383dcd4fe., hostname=bd6592c37bb2,40087,1732486423713, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8ca644d2cada6dc44b363b7383dcd4fe, server=bd6592c37bb2,40087,1732486423713 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T22:16:42,125 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40087 {}] ipc.CallRunner(138): callId: 63 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49002 deadline: 1732486662121, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8ca644d2cada6dc44b363b7383dcd4fe, server=bd6592c37bb2,40087,1732486423713 2024-11-24T22:16:42,125 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732486585339.8ca644d2cada6dc44b363b7383dcd4fe., hostname=bd6592c37bb2,40087,1732486423713, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8ca644d2cada6dc44b363b7383dcd4fe, server=bd6592c37bb2,40087,1732486423713 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T22:16:42,125 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732486585339.8ca644d2cada6dc44b363b7383dcd4fe., hostname=bd6592c37bb2,40087,1732486423713, seqNum=5 because the exception is null or not the one we care about 2024-11-24T22:16:42,125 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732486585339.8ca644d2cada6dc44b363b7383dcd4fe., hostname=bd6592c37bb2,40087,1732486423713, seqNum=5 , the old value is region=TestAcidGuarantees,,1732486585339.8ca644d2cada6dc44b363b7383dcd4fe., hostname=bd6592c37bb2,40087,1732486423713, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8ca644d2cada6dc44b363b7383dcd4fe, server=bd6592c37bb2,40087,1732486423713 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T22:16:42,125 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40087 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8ca644d2cada6dc44b363b7383dcd4fe, server=bd6592c37bb2,40087,1732486423713 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-24T22:16:42,125 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732486585339.8ca644d2cada6dc44b363b7383dcd4fe., hostname=bd6592c37bb2,40087,1732486423713, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8ca644d2cada6dc44b363b7383dcd4fe, server=bd6592c37bb2,40087,1732486423713 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T22:16:42,125 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732486585339.8ca644d2cada6dc44b363b7383dcd4fe., hostname=bd6592c37bb2,40087,1732486423713, seqNum=5 because the exception is null or not the one we care about 2024-11-24T22:16:42,125 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40087 {}] ipc.CallRunner(138): callId: 63 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48986 deadline: 1732486662121, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8ca644d2cada6dc44b363b7383dcd4fe, server=bd6592c37bb2,40087,1732486423713 2024-11-24T22:16:42,125 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732486585339.8ca644d2cada6dc44b363b7383dcd4fe., hostname=bd6592c37bb2,40087,1732486423713, seqNum=5 , the old value is region=TestAcidGuarantees,,1732486585339.8ca644d2cada6dc44b363b7383dcd4fe., hostname=bd6592c37bb2,40087,1732486423713, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8ca644d2cada6dc44b363b7383dcd4fe, server=bd6592c37bb2,40087,1732486423713 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T22:16:42,125 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732486585339.8ca644d2cada6dc44b363b7383dcd4fe., hostname=bd6592c37bb2,40087,1732486423713, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8ca644d2cada6dc44b363b7383dcd4fe, server=bd6592c37bb2,40087,1732486423713 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T22:16:42,125 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732486585339.8ca644d2cada6dc44b363b7383dcd4fe., hostname=bd6592c37bb2,40087,1732486423713, seqNum=5 because the exception is null or not the one we care about 2024-11-24T22:16:42,167 DEBUG [RS:0;bd6592c37bb2:40087-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/8ca644d2cada6dc44b363b7383dcd4fe/.tmp/C/09a368f67227449abf941025f6c40c9f as hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/8ca644d2cada6dc44b363b7383dcd4fe/C/09a368f67227449abf941025f6c40c9f 2024-11-24T22:16:42,170 INFO [RS:0;bd6592c37bb2:40087-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 8ca644d2cada6dc44b363b7383dcd4fe/C of 8ca644d2cada6dc44b363b7383dcd4fe into 09a368f67227449abf941025f6c40c9f(size=12.3 K), total size for store is 12.3 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-24T22:16:42,170 DEBUG [RS:0;bd6592c37bb2:40087-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 8ca644d2cada6dc44b363b7383dcd4fe: 2024-11-24T22:16:42,170 INFO [RS_FLUSH_OPERATIONS-regionserver/bd6592c37bb2:0-2 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=24.60 KB at sequenceid=201 (bloomFilter=true), to=hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/8ca644d2cada6dc44b363b7383dcd4fe/.tmp/C/b6da2cc25fa44492ad282381b9a45c04 2024-11-24T22:16:42,170 INFO [RS:0;bd6592c37bb2:40087-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732486585339.8ca644d2cada6dc44b363b7383dcd4fe., storeName=8ca644d2cada6dc44b363b7383dcd4fe/C, priority=13, startTime=1732486601671; duration=0sec 2024-11-24T22:16:42,170 DEBUG [RS:0;bd6592c37bb2:40087-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-24T22:16:42,170 DEBUG [RS:0;bd6592c37bb2:40087-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 8ca644d2cada6dc44b363b7383dcd4fe:C 2024-11-24T22:16:42,174 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bd6592c37bb2:0-2 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/8ca644d2cada6dc44b363b7383dcd4fe/.tmp/A/b8d8d281b8a04eeb8945f0979f39bfcc as hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/8ca644d2cada6dc44b363b7383dcd4fe/A/b8d8d281b8a04eeb8945f0979f39bfcc 2024-11-24T22:16:42,176 INFO [RS_FLUSH_OPERATIONS-regionserver/bd6592c37bb2:0-2 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/8ca644d2cada6dc44b363b7383dcd4fe/A/b8d8d281b8a04eeb8945f0979f39bfcc, entries=150, sequenceid=201, filesize=30.4 K 2024-11-24T22:16:42,177 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bd6592c37bb2:0-2 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/8ca644d2cada6dc44b363b7383dcd4fe/.tmp/B/0a9cb8108ab84a25a32039517a52cbeb as hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/8ca644d2cada6dc44b363b7383dcd4fe/B/0a9cb8108ab84a25a32039517a52cbeb 2024-11-24T22:16:42,180 INFO [RS_FLUSH_OPERATIONS-regionserver/bd6592c37bb2:0-2 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/8ca644d2cada6dc44b363b7383dcd4fe/B/0a9cb8108ab84a25a32039517a52cbeb, entries=150, sequenceid=201, filesize=11.9 K 2024-11-24T22:16:42,180 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bd6592c37bb2:0-2 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/8ca644d2cada6dc44b363b7383dcd4fe/.tmp/C/b6da2cc25fa44492ad282381b9a45c04 as hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/8ca644d2cada6dc44b363b7383dcd4fe/C/b6da2cc25fa44492ad282381b9a45c04 2024-11-24T22:16:42,184 INFO [RS_FLUSH_OPERATIONS-regionserver/bd6592c37bb2:0-2 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/8ca644d2cada6dc44b363b7383dcd4fe/C/b6da2cc25fa44492ad282381b9a45c04, entries=150, sequenceid=201, filesize=11.9 K 2024-11-24T22:16:42,184 INFO [RS_FLUSH_OPERATIONS-regionserver/bd6592c37bb2:0-2 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.HRegion(3140): Finished flush of dataSize ~73.80 KB/75570, heapSize ~219.84 KB/225120, currentSize=107.34 KB/109920 for 8ca644d2cada6dc44b363b7383dcd4fe in 499ms, sequenceid=201, compaction requested=false 2024-11-24T22:16:42,185 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bd6592c37bb2:0-2 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.HRegion(2603): Flush status journal for 8ca644d2cada6dc44b363b7383dcd4fe: 2024-11-24T22:16:42,185 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bd6592c37bb2:0-2 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732486585339.8ca644d2cada6dc44b363b7383dcd4fe. 2024-11-24T22:16:42,185 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bd6592c37bb2:0-2 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=161 2024-11-24T22:16:42,185 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37701 {}] master.HMaster(4169): Remote procedure done, pid=161 2024-11-24T22:16:42,188 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=161, resume processing ppid=160 2024-11-24T22:16:42,188 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=161, ppid=160, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.1120 sec 2024-11-24T22:16:42,189 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=160, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=160, table=TestAcidGuarantees in 1.1160 sec 2024-11-24T22:16:42,211 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37701 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=160 2024-11-24T22:16:42,211 INFO [RPCClient-NioEventLoopGroup-4-12 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-24T22:16:42,212 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37701 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-24T22:16:42,212 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37701 {}] procedure2.ProcedureExecutor(1139): Stored pid=162, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=162, table=TestAcidGuarantees 2024-11-24T22:16:42,213 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37701 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=162 2024-11-24T22:16:42,213 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=162, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=162, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-24T22:16:42,214 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=162, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=162, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-24T22:16:42,214 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=163, ppid=162, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-24T22:16:42,321 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37701 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=162 2024-11-24T22:16:42,333 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40087 {}] regionserver.HRegion(8855): Flush requested on 8ca644d2cada6dc44b363b7383dcd4fe 2024-11-24T22:16:42,334 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 8ca644d2cada6dc44b363b7383dcd4fe 3/3 column families, dataSize=114.05 KB heapSize=339.42 KB 2024-11-24T22:16:42,338 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 8ca644d2cada6dc44b363b7383dcd4fe, store=A 2024-11-24T22:16:42,338 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/bd6592c37bb2:0-1 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=209, cellSize=9565, totalHeapSize=21976, min timestamp=1732486601981, max timestamp=1732486602334Num uniques 859; 2024-11-24T22:16:42,338 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/bd6592c37bb2:0-9 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=209, cellSize=9565, totalHeapSize=21976, min timestamp=1732486601981, max timestamp=1732486602334Num uniques 859; 2024-11-24T22:16:42,338 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/bd6592c37bb2:0-8 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=209, cellSize=9565, totalHeapSize=21976, min timestamp=1732486601981, max timestamp=1732486602334Num uniques 859; 2024-11-24T22:16:42,338 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/bd6592c37bb2:0-1 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=9565, suffix heap size=117064,new segment heap size=21976 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=209 2024-11-24T22:16:42,338 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/bd6592c37bb2:0-8 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=9565, suffix heap size=117064,new segment heap size=21976 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=209 2024-11-24T22:16:42,338 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/bd6592c37bb2:0-9 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=9565, suffix heap size=117064,new segment heap size=21976 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=209 2024-11-24T22:16:42,338 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-24T22:16:42,338 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 8ca644d2cada6dc44b363b7383dcd4fe, store=B 2024-11-24T22:16:42,338 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-24T22:16:42,338 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 8ca644d2cada6dc44b363b7383dcd4fe, store=C 2024-11-24T22:16:42,338 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-24T22:16:42,354 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411249d703a0b5ce54c4398620884718cbe38_8ca644d2cada6dc44b363b7383dcd4fe is 50, key is test_row_0/A:col10/1732486602334/Put/seqid=0 2024-11-24T22:16:42,360 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45929 is added to blk_1073742442_1618 (size=12304) 2024-11-24T22:16:42,361 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T22:16:42,364 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411249d703a0b5ce54c4398620884718cbe38_8ca644d2cada6dc44b363b7383dcd4fe to hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411249d703a0b5ce54c4398620884718cbe38_8ca644d2cada6dc44b363b7383dcd4fe 2024-11-24T22:16:42,365 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/8ca644d2cada6dc44b363b7383dcd4fe/.tmp/A/53ee59bfb6b54d9f8799663736d3c46a, store: [table=TestAcidGuarantees family=A region=8ca644d2cada6dc44b363b7383dcd4fe] 2024-11-24T22:16:42,365 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/8ca644d2cada6dc44b363b7383dcd4fe/.tmp/A/53ee59bfb6b54d9f8799663736d3c46a is 175, key is test_row_0/A:col10/1732486602334/Put/seqid=0 2024-11-24T22:16:42,366 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=40087 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=163 2024-11-24T22:16:42,366 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bd6592c37bb2:0-0 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732486585339.8ca644d2cada6dc44b363b7383dcd4fe. 2024-11-24T22:16:42,366 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bd6592c37bb2:0-0 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732486585339.8ca644d2cada6dc44b363b7383dcd4fe. as already flushing 2024-11-24T22:16:42,366 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bd6592c37bb2:0-0 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732486585339.8ca644d2cada6dc44b363b7383dcd4fe. 2024-11-24T22:16:42,366 ERROR [RS_FLUSH_OPERATIONS-regionserver/bd6592c37bb2:0-0 {event_type=RS_FLUSH_REGIONS, pid=163}] handler.RSProcedureHandler(58): pid=163 java.io.IOException: Unable to complete flush {ENCODED => 8ca644d2cada6dc44b363b7383dcd4fe, NAME => 'TestAcidGuarantees,,1732486585339.8ca644d2cada6dc44b363b7383dcd4fe.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T22:16:42,367 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bd6592c37bb2:0-0 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=163 java.io.IOException: Unable to complete flush {ENCODED => 8ca644d2cada6dc44b363b7383dcd4fe, NAME => 'TestAcidGuarantees,,1732486585339.8ca644d2cada6dc44b363b7383dcd4fe.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T22:16:42,368 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37701 {}] master.HMaster(4177): Remote procedure failed, pid=163 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 8ca644d2cada6dc44b363b7383dcd4fe, NAME => 'TestAcidGuarantees,,1732486585339.8ca644d2cada6dc44b363b7383dcd4fe.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 8ca644d2cada6dc44b363b7383dcd4fe, NAME => 'TestAcidGuarantees,,1732486585339.8ca644d2cada6dc44b363b7383dcd4fe.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T22:16:42,383 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/bd6592c37bb2:0-6 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=209, cellSize=9565, totalHeapSize=21976, min timestamp=1732486602371, max timestamp=1732486602380Num uniques 859; 2024-11-24T22:16:42,384 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/bd6592c37bb2:0-6 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=9565, suffix heap size=117064,new segment heap size=21976 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=209 2024-11-24T22:16:42,386 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/bd6592c37bb2:0-6 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=209, cellSize=9565, totalHeapSize=21976, min timestamp=1732486602371, max timestamp=1732486602380Num uniques 859; 2024-11-24T22:16:42,386 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/bd6592c37bb2:0-6 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=9565, suffix heap size=117064,new segment heap size=21976 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=209 2024-11-24T22:16:42,389 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/bd6592c37bb2:0-6 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=150, cellSize=6870, totalHeapSize=15840, min timestamp=1732486602371, max timestamp=1732486602380Num uniques 859; 2024-11-24T22:16:42,389 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/bd6592c37bb2:0-6 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=6870, suffix heap size=117064,new segment heap size=15840 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=150 2024-11-24T22:16:42,390 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45929 is added to blk_1073742443_1619 (size=31105) 2024-11-24T22:16:42,425 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/bd6592c37bb2:0-0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=218, cellSize=9978, totalHeapSize=22912, min timestamp=1732486602402, max timestamp=1732486602419Num uniques 1068; 2024-11-24T22:16:42,425 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/bd6592c37bb2:0-0 {}] regionserver.CompactionPipeline(163): Suffix data size=48908, new segment data size=9978, suffix heap size=139040,new segment heap size=22912 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1068, new segment cells count=218 2024-11-24T22:16:42,425 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/bd6592c37bb2:0-5 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=268, cellSize=12268, totalHeapSize=28112, min timestamp=1732486602402, max timestamp=1732486602419Num uniques 1068; 2024-11-24T22:16:42,425 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/bd6592c37bb2:0-5 {}] regionserver.CompactionPipeline(163): Suffix data size=48908, new segment data size=12268, suffix heap size=139040,new segment heap size=28112 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1068, new segment cells count=268 2024-11-24T22:16:42,428 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/bd6592c37bb2:0-0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=168, cellSize=7688, totalHeapSize=17712, min timestamp=1732486602402, max timestamp=1732486602419Num uniques 1009; 2024-11-24T22:16:42,428 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/bd6592c37bb2:0-0 {}] regionserver.CompactionPipeline(163): Suffix data size=46213, new segment data size=7688, suffix heap size=132904,new segment heap size=17712 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1009, new segment cells count=168 2024-11-24T22:16:42,473 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/bd6592c37bb2:0-1 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=227, cellSize=10392, totalHeapSize=23848, min timestamp=1732486602451, max timestamp=1732486602469Num uniques 1077; 2024-11-24T22:16:42,473 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/bd6592c37bb2:0-1 {}] regionserver.CompactionPipeline(163): Suffix data size=49322, new segment data size=10392, suffix heap size=139976,new segment heap size=23848 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1077, new segment cells count=227 2024-11-24T22:16:42,473 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/bd6592c37bb2:0-4 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=227, cellSize=10392, totalHeapSize=23848, min timestamp=1732486602451, max timestamp=1732486602469Num uniques 1127; 2024-11-24T22:16:42,473 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/bd6592c37bb2:0-4 {}] regionserver.CompactionPipeline(163): Suffix data size=51612, new segment data size=10392, suffix heap size=145176,new segment heap size=23848 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1127, new segment cells count=227 2024-11-24T22:16:42,476 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/bd6592c37bb2:0-8 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=177, cellSize=8102, totalHeapSize=18648, min timestamp=1732486602451, max timestamp=1732486602469Num uniques 1027; 2024-11-24T22:16:42,476 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/bd6592c37bb2:0-8 {}] regionserver.CompactionPipeline(163): Suffix data size=47032, new segment data size=8102, suffix heap size=134776,new segment heap size=18648 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1027, new segment cells count=177 2024-11-24T22:16:42,511 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/bd6592c37bb2:0-9 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=236, cellSize=10806, totalHeapSize=24784, min timestamp=1732486602498, max timestamp=1732486602506Num uniques 1086; 2024-11-24T22:16:42,511 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/bd6592c37bb2:0-9 {}] regionserver.CompactionPipeline(163): Suffix data size=49736, new segment data size=10806, suffix heap size=140912,new segment heap size=24784 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1086, new segment cells count=236 2024-11-24T22:16:42,511 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/bd6592c37bb2:0-2 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=236, cellSize=10806, totalHeapSize=24784, min timestamp=1732486602498, max timestamp=1732486602506Num uniques 1086; 2024-11-24T22:16:42,511 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/bd6592c37bb2:0-2 {}] regionserver.CompactionPipeline(163): Suffix data size=49736, new segment data size=10806, suffix heap size=140912,new segment heap size=24784 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1086, new segment cells count=236 2024-11-24T22:16:42,511 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/bd6592c37bb2:0-3 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=236, cellSize=10806, totalHeapSize=24784, min timestamp=1732486602498, max timestamp=1732486602506Num uniques 1036; 2024-11-24T22:16:42,511 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/bd6592c37bb2:0-3 {}] regionserver.CompactionPipeline(163): Suffix data size=47446, new segment data size=10806, suffix heap size=135712,new segment heap size=24784 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1036, new segment cells count=236 2024-11-24T22:16:42,524 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=40087 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=163 2024-11-24T22:16:42,524 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bd6592c37bb2:0-1 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732486585339.8ca644d2cada6dc44b363b7383dcd4fe. 2024-11-24T22:16:42,524 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bd6592c37bb2:0-1 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732486585339.8ca644d2cada6dc44b363b7383dcd4fe. as already flushing 2024-11-24T22:16:42,524 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bd6592c37bb2:0-1 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732486585339.8ca644d2cada6dc44b363b7383dcd4fe. 2024-11-24T22:16:42,524 ERROR [RS_FLUSH_OPERATIONS-regionserver/bd6592c37bb2:0-1 {event_type=RS_FLUSH_REGIONS, pid=163}] handler.RSProcedureHandler(58): pid=163 java.io.IOException: Unable to complete flush {ENCODED => 8ca644d2cada6dc44b363b7383dcd4fe, NAME => 'TestAcidGuarantees,,1732486585339.8ca644d2cada6dc44b363b7383dcd4fe.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T22:16:42,524 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bd6592c37bb2:0-1 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=163 java.io.IOException: Unable to complete flush {ENCODED => 8ca644d2cada6dc44b363b7383dcd4fe, NAME => 'TestAcidGuarantees,,1732486585339.8ca644d2cada6dc44b363b7383dcd4fe.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T22:16:42,525 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37701 {}] master.HMaster(4177): Remote procedure failed, pid=163 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 8ca644d2cada6dc44b363b7383dcd4fe, NAME => 'TestAcidGuarantees,,1732486585339.8ca644d2cada6dc44b363b7383dcd4fe.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 8ca644d2cada6dc44b363b7383dcd4fe, NAME => 'TestAcidGuarantees,,1732486585339.8ca644d2cada6dc44b363b7383dcd4fe.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T22:16:42,532 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37701 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=162 2024-11-24T22:16:42,552 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/bd6592c37bb2:0-6 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=245, cellSize=11220, totalHeapSize=25720, min timestamp=1732486602535, max timestamp=1732486602545Num uniques 1095; 2024-11-24T22:16:42,552 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/bd6592c37bb2:0-7 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=245, cellSize=11220, totalHeapSize=25720, min timestamp=1732486602535, max timestamp=1732486602545Num uniques 1095; 2024-11-24T22:16:42,552 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/bd6592c37bb2:0-7 {}] regionserver.CompactionPipeline(163): Suffix data size=50150, new segment data size=11220, suffix heap size=141848,new segment heap size=25720 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1095, new segment cells count=245 2024-11-24T22:16:42,552 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/bd6592c37bb2:0-6 {}] regionserver.CompactionPipeline(163): Suffix data size=50150, new segment data size=11220, suffix heap size=141848,new segment heap size=25720 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1095, new segment cells count=245 2024-11-24T22:16:42,555 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/bd6592c37bb2:0-5 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=245, cellSize=11220, totalHeapSize=25720, min timestamp=1732486602535, max timestamp=1732486602545Num uniques 1095; 2024-11-24T22:16:42,555 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/bd6592c37bb2:0-5 {}] regionserver.CompactionPipeline(163): Suffix data size=50150, new segment data size=11220, suffix heap size=141848,new segment heap size=25720 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1095, new segment cells count=245 2024-11-24T22:16:42,601 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40087 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8ca644d2cada6dc44b363b7383dcd4fe, server=bd6592c37bb2,40087,1732486423713 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-24T22:16:42,601 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40087 {}] ipc.CallRunner(138): callId: 100 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49002 deadline: 1732486662594, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8ca644d2cada6dc44b363b7383dcd4fe, server=bd6592c37bb2,40087,1732486423713 2024-11-24T22:16:42,604 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/bd6592c37bb2:0-1 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=204, cellSize=9340, totalHeapSize=21456, min timestamp=1732486602581, max timestamp=1732486602600Num uniques 1104; 2024-11-24T22:16:42,604 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/bd6592c37bb2:0-0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=204, cellSize=9340, totalHeapSize=21456, min timestamp=1732486602581, max timestamp=1732486602600Num uniques 1104; 2024-11-24T22:16:42,605 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/bd6592c37bb2:0-4 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=204, cellSize=9340, totalHeapSize=21456, min timestamp=1732486602581, max timestamp=1732486602600Num uniques 1104; 2024-11-24T22:16:42,605 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/bd6592c37bb2:0-1 {}] regionserver.CompactionPipeline(163): Suffix data size=50560, new segment data size=9340, suffix heap size=142784,new segment heap size=21456 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1104, new segment cells count=204 2024-11-24T22:16:42,605 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/bd6592c37bb2:0-4 {}] regionserver.CompactionPipeline(163): Suffix data size=50560, new segment data size=9340, suffix heap size=142784,new segment heap size=21456 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1104, new segment cells count=204 2024-11-24T22:16:42,605 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/bd6592c37bb2:0-0 {}] regionserver.CompactionPipeline(163): Suffix data size=50560, new segment data size=9340, suffix heap size=142784,new segment heap size=21456 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1104, new segment cells count=204 2024-11-24T22:16:42,605 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732486585339.8ca644d2cada6dc44b363b7383dcd4fe., hostname=bd6592c37bb2,40087,1732486423713, seqNum=5 , the old value is region=TestAcidGuarantees,,1732486585339.8ca644d2cada6dc44b363b7383dcd4fe., hostname=bd6592c37bb2,40087,1732486423713, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8ca644d2cada6dc44b363b7383dcd4fe, server=bd6592c37bb2,40087,1732486423713 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T22:16:42,605 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732486585339.8ca644d2cada6dc44b363b7383dcd4fe., hostname=bd6592c37bb2,40087,1732486423713, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8ca644d2cada6dc44b363b7383dcd4fe, server=bd6592c37bb2,40087,1732486423713 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T22:16:42,605 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732486585339.8ca644d2cada6dc44b363b7383dcd4fe., hostname=bd6592c37bb2,40087,1732486423713, seqNum=5 because the exception is null or not the one we care about 2024-11-24T22:16:42,663 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/bd6592c37bb2:0-8 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=163, cellSize=7458, totalHeapSize=17192, min timestamp=1732486602645, max timestamp=1732486602658Num uniques 1063; 2024-11-24T22:16:42,663 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/bd6592c37bb2:0-8 {}] regionserver.CompactionPipeline(163): Suffix data size=48678, new segment data size=7458, suffix heap size=138520,new segment heap size=17192 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1063, new segment cells count=163 2024-11-24T22:16:42,663 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/bd6592c37bb2:0-2 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=163, cellSize=7458, totalHeapSize=17192, min timestamp=1732486602645, max timestamp=1732486602658Num uniques 1063; 2024-11-24T22:16:42,663 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/bd6592c37bb2:0-9 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=163, cellSize=7458, totalHeapSize=17192, min timestamp=1732486602645, max timestamp=1732486602658Num uniques 1063; 2024-11-24T22:16:42,663 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/bd6592c37bb2:0-2 {}] regionserver.CompactionPipeline(163): Suffix data size=48678, new segment data size=7458, suffix heap size=138520,new segment heap size=17192 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1063, new segment cells count=163 2024-11-24T22:16:42,663 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/bd6592c37bb2:0-9 {}] regionserver.CompactionPipeline(163): Suffix data size=48678, new segment data size=7458, suffix heap size=138520,new segment heap size=17192 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1063, new segment cells count=163 2024-11-24T22:16:42,676 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=40087 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=163 2024-11-24T22:16:42,676 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bd6592c37bb2:0-2 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732486585339.8ca644d2cada6dc44b363b7383dcd4fe. 2024-11-24T22:16:42,676 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bd6592c37bb2:0-2 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732486585339.8ca644d2cada6dc44b363b7383dcd4fe. as already flushing 2024-11-24T22:16:42,676 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bd6592c37bb2:0-2 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732486585339.8ca644d2cada6dc44b363b7383dcd4fe. 2024-11-24T22:16:42,676 ERROR [RS_FLUSH_OPERATIONS-regionserver/bd6592c37bb2:0-2 {event_type=RS_FLUSH_REGIONS, pid=163}] handler.RSProcedureHandler(58): pid=163 java.io.IOException: Unable to complete flush {ENCODED => 8ca644d2cada6dc44b363b7383dcd4fe, NAME => 'TestAcidGuarantees,,1732486585339.8ca644d2cada6dc44b363b7383dcd4fe.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T22:16:42,677 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bd6592c37bb2:0-2 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=163 java.io.IOException: Unable to complete flush {ENCODED => 8ca644d2cada6dc44b363b7383dcd4fe, NAME => 'TestAcidGuarantees,,1732486585339.8ca644d2cada6dc44b363b7383dcd4fe.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T22:16:42,677 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37701 {}] master.HMaster(4177): Remote procedure failed, pid=163 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 8ca644d2cada6dc44b363b7383dcd4fe, NAME => 'TestAcidGuarantees,,1732486585339.8ca644d2cada6dc44b363b7383dcd4fe.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 8ca644d2cada6dc44b363b7383dcd4fe, NAME => 'TestAcidGuarantees,,1732486585339.8ca644d2cada6dc44b363b7383dcd4fe.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T22:16:42,717 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/bd6592c37bb2:0-3 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=172, cellSize=7872, totalHeapSize=18128, min timestamp=1732486602700, max timestamp=1732486602711Num uniques 1022; 2024-11-24T22:16:42,717 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/bd6592c37bb2:0-3 {}] regionserver.CompactionPipeline(163): Suffix data size=46802, new segment data size=7872, suffix heap size=134256,new segment heap size=18128 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1022, new segment cells count=172 2024-11-24T22:16:42,717 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/bd6592c37bb2:0-6 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=172, cellSize=7872, totalHeapSize=18128, min timestamp=1732486602700, max timestamp=1732486602711Num uniques 1022; 2024-11-24T22:16:42,717 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/bd6592c37bb2:0-7 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=172, cellSize=7872, totalHeapSize=18128, min timestamp=1732486602700, max timestamp=1732486602711Num uniques 1022; 2024-11-24T22:16:42,717 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/bd6592c37bb2:0-6 {}] regionserver.CompactionPipeline(163): Suffix data size=46802, new segment data size=7872, suffix heap size=134256,new segment heap size=18128 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1022, new segment cells count=172 2024-11-24T22:16:42,717 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/bd6592c37bb2:0-7 {}] regionserver.CompactionPipeline(163): Suffix data size=46802, new segment data size=7872, suffix heap size=134256,new segment heap size=18128 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1022, new segment cells count=172 2024-11-24T22:16:42,754 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/bd6592c37bb2:0-5 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=231, cellSize=10576, totalHeapSize=24264, min timestamp=1732486602740, max timestamp=1732486602749Num uniques 1031; 2024-11-24T22:16:42,754 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/bd6592c37bb2:0-5 {}] regionserver.CompactionPipeline(163): Suffix data size=47216, new segment data size=10576, suffix heap size=135192,new segment heap size=24264 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1031, new segment cells count=231 2024-11-24T22:16:42,754 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/bd6592c37bb2:0-1 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=231, cellSize=10576, totalHeapSize=24264, min timestamp=1732486602740, max timestamp=1732486602749Num uniques 1031; 2024-11-24T22:16:42,754 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/bd6592c37bb2:0-1 {}] regionserver.CompactionPipeline(163): Suffix data size=47216, new segment data size=10576, suffix heap size=135192,new segment heap size=24264 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1031, new segment cells count=231 2024-11-24T22:16:42,754 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/bd6592c37bb2:0-4 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=231, cellSize=10576, totalHeapSize=24264, min timestamp=1732486602740, max timestamp=1732486602749Num uniques 1031; 2024-11-24T22:16:42,754 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/bd6592c37bb2:0-4 {}] regionserver.CompactionPipeline(163): Suffix data size=47216, new segment data size=10576, suffix heap size=135192,new segment heap size=24264 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1031, new segment cells count=231 2024-11-24T22:16:42,790 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=225, memsize=11.2 K, hasBloomFilter=true, into tmp file hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/8ca644d2cada6dc44b363b7383dcd4fe/.tmp/A/53ee59bfb6b54d9f8799663736d3c46a 2024-11-24T22:16:42,797 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/bd6592c37bb2:0-0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=290, cellSize=13280, totalHeapSize=30400, min timestamp=1732486602766, max timestamp=1732486602792Num uniques 1090; 2024-11-24T22:16:42,797 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/bd6592c37bb2:0-2 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=290, cellSize=13280, totalHeapSize=30400, min timestamp=1732486602766, max timestamp=1732486602792Num uniques 1090; 2024-11-24T22:16:42,798 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/bd6592c37bb2:0-0 {}] regionserver.CompactionPipeline(163): Suffix data size=49920, new segment data size=13280, suffix heap size=141328,new segment heap size=30400 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1090, new segment cells count=290 2024-11-24T22:16:42,798 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/bd6592c37bb2:0-2 {}] regionserver.CompactionPipeline(163): Suffix data size=49920, new segment data size=13280, suffix heap size=141328,new segment heap size=30400 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1090, new segment cells count=290 2024-11-24T22:16:42,798 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/bd6592c37bb2:0-8 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=290, cellSize=13280, totalHeapSize=30400, min timestamp=1732486602766, max timestamp=1732486602792Num uniques 1090; 2024-11-24T22:16:42,798 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/bd6592c37bb2:0-8 {}] regionserver.CompactionPipeline(163): Suffix data size=49920, new segment data size=13280, suffix heap size=141328,new segment heap size=30400 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1090, new segment cells count=290 2024-11-24T22:16:42,800 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/8ca644d2cada6dc44b363b7383dcd4fe/.tmp/B/63e6053a21424b89a81ca99ff0a18861 is 50, key is test_row_0/B:col10/1732486602334/Put/seqid=0 2024-11-24T22:16:42,806 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45929 is added to blk_1073742444_1620 (size=12151) 2024-11-24T22:16:42,829 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=40087 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=163 2024-11-24T22:16:42,829 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bd6592c37bb2:0-0 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732486585339.8ca644d2cada6dc44b363b7383dcd4fe. 2024-11-24T22:16:42,829 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bd6592c37bb2:0-0 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732486585339.8ca644d2cada6dc44b363b7383dcd4fe. as already flushing 2024-11-24T22:16:42,829 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bd6592c37bb2:0-0 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732486585339.8ca644d2cada6dc44b363b7383dcd4fe. 2024-11-24T22:16:42,829 ERROR [RS_FLUSH_OPERATIONS-regionserver/bd6592c37bb2:0-0 {event_type=RS_FLUSH_REGIONS, pid=163}] handler.RSProcedureHandler(58): pid=163 java.io.IOException: Unable to complete flush {ENCODED => 8ca644d2cada6dc44b363b7383dcd4fe, NAME => 'TestAcidGuarantees,,1732486585339.8ca644d2cada6dc44b363b7383dcd4fe.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T22:16:42,829 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bd6592c37bb2:0-0 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=163 java.io.IOException: Unable to complete flush {ENCODED => 8ca644d2cada6dc44b363b7383dcd4fe, NAME => 'TestAcidGuarantees,,1732486585339.8ca644d2cada6dc44b363b7383dcd4fe.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T22:16:42,830 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37701 {}] master.HMaster(4177): Remote procedure failed, pid=163 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 8ca644d2cada6dc44b363b7383dcd4fe, NAME => 'TestAcidGuarantees,,1732486585339.8ca644d2cada6dc44b363b7383dcd4fe.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 8ca644d2cada6dc44b363b7383dcd4fe, NAME => 'TestAcidGuarantees,,1732486585339.8ca644d2cada6dc44b363b7383dcd4fe.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T22:16:42,839 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40087 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8ca644d2cada6dc44b363b7383dcd4fe, server=bd6592c37bb2,40087,1732486423713 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-24T22:16:42,839 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40087 {}] ipc.CallRunner(138): callId: 118 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49002 deadline: 1732486662833, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8ca644d2cada6dc44b363b7383dcd4fe, server=bd6592c37bb2,40087,1732486423713 2024-11-24T22:16:42,839 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732486585339.8ca644d2cada6dc44b363b7383dcd4fe., hostname=bd6592c37bb2,40087,1732486423713, seqNum=5 , the old value is region=TestAcidGuarantees,,1732486585339.8ca644d2cada6dc44b363b7383dcd4fe., hostname=bd6592c37bb2,40087,1732486423713, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8ca644d2cada6dc44b363b7383dcd4fe, server=bd6592c37bb2,40087,1732486423713 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T22:16:42,839 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732486585339.8ca644d2cada6dc44b363b7383dcd4fe., hostname=bd6592c37bb2,40087,1732486423713, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8ca644d2cada6dc44b363b7383dcd4fe, server=bd6592c37bb2,40087,1732486423713 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T22:16:42,839 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732486585339.8ca644d2cada6dc44b363b7383dcd4fe., hostname=bd6592c37bb2,40087,1732486423713, seqNum=5 because the exception is null or not the one we care about 2024-11-24T22:16:42,840 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37701 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=162 2024-11-24T22:16:42,842 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/bd6592c37bb2:0-3 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=199, cellSize=9108, totalHeapSize=20936, min timestamp=1732486602825, max timestamp=1732486602838Num uniques 1149; 2024-11-24T22:16:42,842 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/bd6592c37bb2:0-3 {}] regionserver.CompactionPipeline(163): Suffix data size=52618, new segment data size=9108, suffix heap size=147464,new segment heap size=20936 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1149, new segment cells count=199 2024-11-24T22:16:42,843 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/bd6592c37bb2:0-9 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=199, cellSize=9114, totalHeapSize=20936, min timestamp=1732486602825, max timestamp=1732486602837Num uniques 1149; 2024-11-24T22:16:42,843 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/bd6592c37bb2:0-9 {}] regionserver.CompactionPipeline(163): Suffix data size=52624, new segment data size=9114, suffix heap size=147464,new segment heap size=20936 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1149, new segment cells count=199 2024-11-24T22:16:42,843 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/bd6592c37bb2:0-6 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=199, cellSize=9104, totalHeapSize=20936, min timestamp=1732486602825, max timestamp=1732486602838Num uniques 1149; 2024-11-24T22:16:42,843 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/bd6592c37bb2:0-6 {}] regionserver.CompactionPipeline(163): Suffix data size=52614, new segment data size=9104, suffix heap size=147464,new segment heap size=20936 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1149, new segment cells count=199 2024-11-24T22:16:42,922 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40087 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8ca644d2cada6dc44b363b7383dcd4fe, server=bd6592c37bb2,40087,1732486423713 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-24T22:16:42,922 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40087 {}] ipc.CallRunner(138): callId: 150 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49050 deadline: 1732486662921, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8ca644d2cada6dc44b363b7383dcd4fe, server=bd6592c37bb2,40087,1732486423713 2024-11-24T22:16:42,923 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732486585339.8ca644d2cada6dc44b363b7383dcd4fe., hostname=bd6592c37bb2,40087,1732486423713, seqNum=5 , the old value is region=TestAcidGuarantees,,1732486585339.8ca644d2cada6dc44b363b7383dcd4fe., hostname=bd6592c37bb2,40087,1732486423713, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8ca644d2cada6dc44b363b7383dcd4fe, server=bd6592c37bb2,40087,1732486423713 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T22:16:42,923 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732486585339.8ca644d2cada6dc44b363b7383dcd4fe., hostname=bd6592c37bb2,40087,1732486423713, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8ca644d2cada6dc44b363b7383dcd4fe, server=bd6592c37bb2,40087,1732486423713 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T22:16:42,923 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732486585339.8ca644d2cada6dc44b363b7383dcd4fe., hostname=bd6592c37bb2,40087,1732486423713, seqNum=5 because the exception is null or not the one we care about 2024-11-24T22:16:42,924 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/bd6592c37bb2:0-1 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=158, cellSize=7230, totalHeapSize=16672, min timestamp=1732486602892, max timestamp=1732486602919Num uniques 1058; 2024-11-24T22:16:42,924 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/bd6592c37bb2:0-1 {}] regionserver.CompactionPipeline(163): Suffix data size=48450, new segment data size=7230, suffix heap size=138000,new segment heap size=16672 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1058, new segment cells count=158 2024-11-24T22:16:42,924 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/bd6592c37bb2:0-7 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=158, cellSize=7230, totalHeapSize=16672, min timestamp=1732486602892, max timestamp=1732486602919Num uniques 1058; 2024-11-24T22:16:42,924 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/bd6592c37bb2:0-5 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=158, cellSize=7230, totalHeapSize=16672, min timestamp=1732486602892, max timestamp=1732486602919Num uniques 1058; 2024-11-24T22:16:42,924 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/bd6592c37bb2:0-7 {}] regionserver.CompactionPipeline(163): Suffix data size=48450, new segment data size=7230, suffix heap size=138000,new segment heap size=16672 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1058, new segment cells count=158 2024-11-24T22:16:42,924 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/bd6592c37bb2:0-5 {}] regionserver.CompactionPipeline(163): Suffix data size=48450, new segment data size=7230, suffix heap size=138000,new segment heap size=16672 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1058, new segment cells count=158 2024-11-24T22:16:42,982 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=40087 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=163 2024-11-24T22:16:42,982 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bd6592c37bb2:0-1 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732486585339.8ca644d2cada6dc44b363b7383dcd4fe. 2024-11-24T22:16:42,982 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bd6592c37bb2:0-1 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732486585339.8ca644d2cada6dc44b363b7383dcd4fe. as already flushing 2024-11-24T22:16:42,982 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bd6592c37bb2:0-1 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732486585339.8ca644d2cada6dc44b363b7383dcd4fe. 2024-11-24T22:16:42,982 ERROR [RS_FLUSH_OPERATIONS-regionserver/bd6592c37bb2:0-1 {event_type=RS_FLUSH_REGIONS, pid=163}] handler.RSProcedureHandler(58): pid=163 java.io.IOException: Unable to complete flush {ENCODED => 8ca644d2cada6dc44b363b7383dcd4fe, NAME => 'TestAcidGuarantees,,1732486585339.8ca644d2cada6dc44b363b7383dcd4fe.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T22:16:42,982 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bd6592c37bb2:0-1 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=163 java.io.IOException: Unable to complete flush {ENCODED => 8ca644d2cada6dc44b363b7383dcd4fe, NAME => 'TestAcidGuarantees,,1732486585339.8ca644d2cada6dc44b363b7383dcd4fe.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T22:16:42,983 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37701 {}] master.HMaster(4177): Remote procedure failed, pid=163 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 8ca644d2cada6dc44b363b7383dcd4fe, NAME => 'TestAcidGuarantees,,1732486585339.8ca644d2cada6dc44b363b7383dcd4fe.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 8ca644d2cada6dc44b363b7383dcd4fe, NAME => 'TestAcidGuarantees,,1732486585339.8ca644d2cada6dc44b363b7383dcd4fe.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T22:16:42,988 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/bd6592c37bb2:0-0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=217, cellSize=9932, totalHeapSize=22808, min timestamp=1732486602947, max timestamp=1732486602983Num uniques 1017; 2024-11-24T22:16:42,988 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/bd6592c37bb2:0-0 {}] regionserver.CompactionPipeline(163): Suffix data size=46572, new segment data size=9932, suffix heap size=133736,new segment heap size=22808 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1017, new segment cells count=217 2024-11-24T22:16:42,988 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/bd6592c37bb2:0-4 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=217, cellSize=9932, totalHeapSize=22808, min timestamp=1732486602947, max timestamp=1732486602983Num uniques 1017; 2024-11-24T22:16:42,989 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/bd6592c37bb2:0-4 {}] regionserver.CompactionPipeline(163): Suffix data size=46572, new segment data size=9932, suffix heap size=133736,new segment heap size=22808 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1017, new segment cells count=217 2024-11-24T22:16:42,989 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/bd6592c37bb2:0-2 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=217, cellSize=9932, totalHeapSize=22808, min timestamp=1732486602947, max timestamp=1732486602983Num uniques 1017; 2024-11-24T22:16:42,989 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/bd6592c37bb2:0-2 {}] regionserver.CompactionPipeline(163): Suffix data size=46572, new segment data size=9932, suffix heap size=133736,new segment heap size=22808 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1017, new segment cells count=217 2024-11-24T22:16:43,021 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40087 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8ca644d2cada6dc44b363b7383dcd4fe, server=bd6592c37bb2,40087,1732486423713 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-24T22:16:43,021 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40087 {}] ipc.CallRunner(138): callId: 151 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48986 deadline: 1732486663015, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8ca644d2cada6dc44b363b7383dcd4fe, server=bd6592c37bb2,40087,1732486423713 2024-11-24T22:16:43,021 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732486585339.8ca644d2cada6dc44b363b7383dcd4fe., hostname=bd6592c37bb2,40087,1732486423713, seqNum=5 , the old value is region=TestAcidGuarantees,,1732486585339.8ca644d2cada6dc44b363b7383dcd4fe., hostname=bd6592c37bb2,40087,1732486423713, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8ca644d2cada6dc44b363b7383dcd4fe, server=bd6592c37bb2,40087,1732486423713 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T22:16:43,021 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732486585339.8ca644d2cada6dc44b363b7383dcd4fe., hostname=bd6592c37bb2,40087,1732486423713, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8ca644d2cada6dc44b363b7383dcd4fe, server=bd6592c37bb2,40087,1732486423713 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T22:16:43,021 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732486585339.8ca644d2cada6dc44b363b7383dcd4fe., hostname=bd6592c37bb2,40087,1732486423713, seqNum=5 because the exception is null or not the one we care about 2024-11-24T22:16:43,023 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/bd6592c37bb2:0-8 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=226, cellSize=10346, totalHeapSize=23744, min timestamp=1732486603007, max timestamp=1732486603018Num uniques 1076; 2024-11-24T22:16:43,023 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/bd6592c37bb2:0-8 {}] regionserver.CompactionPipeline(163): Suffix data size=49276, new segment data size=10346, suffix heap size=139872,new segment heap size=23744 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1076, new segment cells count=226 2024-11-24T22:16:43,024 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/bd6592c37bb2:0-3 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=226, cellSize=10346, totalHeapSize=23744, min timestamp=1732486603007, max timestamp=1732486603018Num uniques 1076; 2024-11-24T22:16:43,024 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/bd6592c37bb2:0-3 {}] regionserver.CompactionPipeline(163): Suffix data size=49276, new segment data size=10346, suffix heap size=139872,new segment heap size=23744 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1076, new segment cells count=226 2024-11-24T22:16:43,024 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/bd6592c37bb2:0-9 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=226, cellSize=10346, totalHeapSize=23744, min timestamp=1732486603007, max timestamp=1732486603018Num uniques 1076; 2024-11-24T22:16:43,024 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/bd6592c37bb2:0-9 {}] regionserver.CompactionPipeline(163): Suffix data size=49276, new segment data size=10346, suffix heap size=139872,new segment heap size=23744 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1076, new segment cells count=226 2024-11-24T22:16:43,085 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/bd6592c37bb2:0-6 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=235, cellSize=10760, totalHeapSize=24680, min timestamp=1732486603043, max timestamp=1732486603080Num uniques 1085; 2024-11-24T22:16:43,085 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/bd6592c37bb2:0-6 {}] regionserver.CompactionPipeline(163): Suffix data size=49690, new segment data size=10760, suffix heap size=140808,new segment heap size=24680 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1085, new segment cells count=235 2024-11-24T22:16:43,085 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/bd6592c37bb2:0-7 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=235, cellSize=10760, totalHeapSize=24680, min timestamp=1732486603043, max timestamp=1732486603080Num uniques 1085; 2024-11-24T22:16:43,085 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/bd6592c37bb2:0-1 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=235, cellSize=10760, totalHeapSize=24680, min timestamp=1732486603043, max timestamp=1732486603080Num uniques 1085; 2024-11-24T22:16:43,085 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/bd6592c37bb2:0-7 {}] regionserver.CompactionPipeline(163): Suffix data size=49690, new segment data size=10760, suffix heap size=140808,new segment heap size=24680 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1085, new segment cells count=235 2024-11-24T22:16:43,085 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/bd6592c37bb2:0-1 {}] regionserver.CompactionPipeline(163): Suffix data size=49690, new segment data size=10760, suffix heap size=140808,new segment heap size=24680 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1085, new segment cells count=235 2024-11-24T22:16:43,134 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=40087 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=163 2024-11-24T22:16:43,134 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bd6592c37bb2:0-2 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732486585339.8ca644d2cada6dc44b363b7383dcd4fe. 2024-11-24T22:16:43,134 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bd6592c37bb2:0-2 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732486585339.8ca644d2cada6dc44b363b7383dcd4fe. as already flushing 2024-11-24T22:16:43,135 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bd6592c37bb2:0-2 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732486585339.8ca644d2cada6dc44b363b7383dcd4fe. 2024-11-24T22:16:43,135 ERROR [RS_FLUSH_OPERATIONS-regionserver/bd6592c37bb2:0-2 {event_type=RS_FLUSH_REGIONS, pid=163}] handler.RSProcedureHandler(58): pid=163 java.io.IOException: Unable to complete flush {ENCODED => 8ca644d2cada6dc44b363b7383dcd4fe, NAME => 'TestAcidGuarantees,,1732486585339.8ca644d2cada6dc44b363b7383dcd4fe.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T22:16:43,135 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bd6592c37bb2:0-2 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=163 java.io.IOException: Unable to complete flush {ENCODED => 8ca644d2cada6dc44b363b7383dcd4fe, NAME => 'TestAcidGuarantees,,1732486585339.8ca644d2cada6dc44b363b7383dcd4fe.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T22:16:43,135 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37701 {}] master.HMaster(4177): Remote procedure failed, pid=163 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 8ca644d2cada6dc44b363b7383dcd4fe, NAME => 'TestAcidGuarantees,,1732486585339.8ca644d2cada6dc44b363b7383dcd4fe.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 8ca644d2cada6dc44b363b7383dcd4fe, NAME => 'TestAcidGuarantees,,1732486585339.8ca644d2cada6dc44b363b7383dcd4fe.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T22:16:43,141 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/bd6592c37bb2:0-5 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=294, cellSize=13464, totalHeapSize=30816, min timestamp=1732486603096, max timestamp=1732486603114Num uniques 1094; 2024-11-24T22:16:43,141 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/bd6592c37bb2:0-5 {}] regionserver.CompactionPipeline(163): Suffix data size=50104, new segment data size=13464, suffix heap size=141744,new segment heap size=30816 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1094, new segment cells count=294 2024-11-24T22:16:43,141 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/bd6592c37bb2:0-4 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=194, cellSize=8884, totalHeapSize=20416, min timestamp=1732486603096, max timestamp=1732486603114Num uniques 1094; 2024-11-24T22:16:43,141 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/bd6592c37bb2:0-4 {}] regionserver.CompactionPipeline(163): Suffix data size=50104, new segment data size=8884, suffix heap size=141744,new segment heap size=20416 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1094, new segment cells count=194 2024-11-24T22:16:43,141 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/bd6592c37bb2:0-0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=294, cellSize=13464, totalHeapSize=30816, min timestamp=1732486603096, max timestamp=1732486603114Num uniques 1094; 2024-11-24T22:16:43,141 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/bd6592c37bb2:0-0 {}] regionserver.CompactionPipeline(163): Suffix data size=50104, new segment data size=13464, suffix heap size=141744,new segment heap size=30816 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1094, new segment cells count=294 2024-11-24T22:16:43,172 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40087 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8ca644d2cada6dc44b363b7383dcd4fe, server=bd6592c37bb2,40087,1732486423713 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-24T22:16:43,172 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40087 {}] ipc.CallRunner(138): callId: 69 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49060 deadline: 1732486663165, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8ca644d2cada6dc44b363b7383dcd4fe, server=bd6592c37bb2,40087,1732486423713 2024-11-24T22:16:43,172 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40087 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8ca644d2cada6dc44b363b7383dcd4fe, server=bd6592c37bb2,40087,1732486423713 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-24T22:16:43,172 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40087 {}] ipc.CallRunner(138): callId: 138 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49002 deadline: 1732486663166, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8ca644d2cada6dc44b363b7383dcd4fe, server=bd6592c37bb2,40087,1732486423713 2024-11-24T22:16:43,172 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40087 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8ca644d2cada6dc44b363b7383dcd4fe, server=bd6592c37bb2,40087,1732486423713 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-24T22:16:43,172 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40087 {}] ipc.CallRunner(138): callId: 156 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48986 deadline: 1732486663167, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8ca644d2cada6dc44b363b7383dcd4fe, server=bd6592c37bb2,40087,1732486423713 2024-11-24T22:16:43,173 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40087 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8ca644d2cada6dc44b363b7383dcd4fe, server=bd6592c37bb2,40087,1732486423713 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-24T22:16:43,173 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40087 {}] ipc.CallRunner(138): callId: 162 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49050 deadline: 1732486663172, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8ca644d2cada6dc44b363b7383dcd4fe, server=bd6592c37bb2,40087,1732486423713 2024-11-24T22:16:43,173 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40087 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8ca644d2cada6dc44b363b7383dcd4fe, server=bd6592c37bb2,40087,1732486423713 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-24T22:16:43,173 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732486585339.8ca644d2cada6dc44b363b7383dcd4fe., hostname=bd6592c37bb2,40087,1732486423713, seqNum=5 , the old value is region=TestAcidGuarantees,,1732486585339.8ca644d2cada6dc44b363b7383dcd4fe., hostname=bd6592c37bb2,40087,1732486423713, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8ca644d2cada6dc44b363b7383dcd4fe, server=bd6592c37bb2,40087,1732486423713 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T22:16:43,173 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732486585339.8ca644d2cada6dc44b363b7383dcd4fe., hostname=bd6592c37bb2,40087,1732486423713, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8ca644d2cada6dc44b363b7383dcd4fe, server=bd6592c37bb2,40087,1732486423713 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T22:16:43,173 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40087 {}] ipc.CallRunner(138): callId: 77 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49026 deadline: 1732486663172, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8ca644d2cada6dc44b363b7383dcd4fe, server=bd6592c37bb2,40087,1732486423713 2024-11-24T22:16:43,173 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732486585339.8ca644d2cada6dc44b363b7383dcd4fe., hostname=bd6592c37bb2,40087,1732486423713, seqNum=5 because the exception is null or not the one we care about 2024-11-24T22:16:43,173 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732486585339.8ca644d2cada6dc44b363b7383dcd4fe., hostname=bd6592c37bb2,40087,1732486423713, seqNum=5 , the old value is region=TestAcidGuarantees,,1732486585339.8ca644d2cada6dc44b363b7383dcd4fe., hostname=bd6592c37bb2,40087,1732486423713, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8ca644d2cada6dc44b363b7383dcd4fe, server=bd6592c37bb2,40087,1732486423713 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T22:16:43,173 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732486585339.8ca644d2cada6dc44b363b7383dcd4fe., hostname=bd6592c37bb2,40087,1732486423713, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8ca644d2cada6dc44b363b7383dcd4fe, server=bd6592c37bb2,40087,1732486423713 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T22:16:43,173 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732486585339.8ca644d2cada6dc44b363b7383dcd4fe., hostname=bd6592c37bb2,40087,1732486423713, seqNum=5 because the exception is null or not the one we care about 2024-11-24T22:16:43,173 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732486585339.8ca644d2cada6dc44b363b7383dcd4fe., hostname=bd6592c37bb2,40087,1732486423713, seqNum=5 , the old value is region=TestAcidGuarantees,,1732486585339.8ca644d2cada6dc44b363b7383dcd4fe., hostname=bd6592c37bb2,40087,1732486423713, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8ca644d2cada6dc44b363b7383dcd4fe, server=bd6592c37bb2,40087,1732486423713 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T22:16:43,174 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732486585339.8ca644d2cada6dc44b363b7383dcd4fe., hostname=bd6592c37bb2,40087,1732486423713, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8ca644d2cada6dc44b363b7383dcd4fe, server=bd6592c37bb2,40087,1732486423713 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T22:16:43,174 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732486585339.8ca644d2cada6dc44b363b7383dcd4fe., hostname=bd6592c37bb2,40087,1732486423713, seqNum=5 because the exception is null or not the one we care about 2024-11-24T22:16:43,174 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732486585339.8ca644d2cada6dc44b363b7383dcd4fe., hostname=bd6592c37bb2,40087,1732486423713, seqNum=5 , the old value is region=TestAcidGuarantees,,1732486585339.8ca644d2cada6dc44b363b7383dcd4fe., hostname=bd6592c37bb2,40087,1732486423713, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8ca644d2cada6dc44b363b7383dcd4fe, server=bd6592c37bb2,40087,1732486423713 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T22:16:43,174 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732486585339.8ca644d2cada6dc44b363b7383dcd4fe., hostname=bd6592c37bb2,40087,1732486423713, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8ca644d2cada6dc44b363b7383dcd4fe, server=bd6592c37bb2,40087,1732486423713 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T22:16:43,174 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732486585339.8ca644d2cada6dc44b363b7383dcd4fe., hostname=bd6592c37bb2,40087,1732486423713, seqNum=5 because the exception is null or not the one we care about 2024-11-24T22:16:43,174 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732486585339.8ca644d2cada6dc44b363b7383dcd4fe., hostname=bd6592c37bb2,40087,1732486423713, seqNum=5 , the old value is region=TestAcidGuarantees,,1732486585339.8ca644d2cada6dc44b363b7383dcd4fe., hostname=bd6592c37bb2,40087,1732486423713, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8ca644d2cada6dc44b363b7383dcd4fe, server=bd6592c37bb2,40087,1732486423713 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T22:16:43,174 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732486585339.8ca644d2cada6dc44b363b7383dcd4fe., hostname=bd6592c37bb2,40087,1732486423713, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8ca644d2cada6dc44b363b7383dcd4fe, server=bd6592c37bb2,40087,1732486423713 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T22:16:43,174 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732486585339.8ca644d2cada6dc44b363b7383dcd4fe., hostname=bd6592c37bb2,40087,1732486423713, seqNum=5 because the exception is null or not the one we care about 2024-11-24T22:16:43,175 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/bd6592c37bb2:0-8 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=153, cellSize=7005, totalHeapSize=16152, min timestamp=1732486603163, max timestamp=1732486603170Num uniques 1153; 2024-11-24T22:16:43,175 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/bd6592c37bb2:0-8 {}] regionserver.CompactionPipeline(163): Suffix data size=52805, new segment data size=7005, suffix heap size=147880,new segment heap size=16152 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1153, new segment cells count=153 2024-11-24T22:16:43,175 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/bd6592c37bb2:0-2 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=153, cellSize=7005, totalHeapSize=16152, min timestamp=1732486603163, max timestamp=1732486603170Num uniques 1153; 2024-11-24T22:16:43,175 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/bd6592c37bb2:0-2 {}] regionserver.CompactionPipeline(163): Suffix data size=52805, new segment data size=7005, suffix heap size=147880,new segment heap size=16152 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1153, new segment cells count=153 2024-11-24T22:16:43,175 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/bd6592c37bb2:0-3 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=153, cellSize=7005, totalHeapSize=16152, min timestamp=1732486603163, max timestamp=1732486603170Num uniques 1053; 2024-11-24T22:16:43,176 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/bd6592c37bb2:0-3 {}] regionserver.CompactionPipeline(163): Suffix data size=48225, new segment data size=7005, suffix heap size=137480,new segment heap size=16152 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1053, new segment cells count=153 2024-11-24T22:16:43,206 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=11.18 KB at sequenceid=225 (bloomFilter=true), to=hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/8ca644d2cada6dc44b363b7383dcd4fe/.tmp/B/63e6053a21424b89a81ca99ff0a18861 2024-11-24T22:16:43,213 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/8ca644d2cada6dc44b363b7383dcd4fe/.tmp/C/c86f7cc086c0485b914cc3b9e5711865 is 50, key is test_row_0/C:col10/1732486602334/Put/seqid=0 2024-11-24T22:16:43,216 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45929 is added to blk_1073742445_1621 (size=12151) 2024-11-24T22:16:43,287 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=40087 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=163 2024-11-24T22:16:43,287 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bd6592c37bb2:0-0 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732486585339.8ca644d2cada6dc44b363b7383dcd4fe. 2024-11-24T22:16:43,287 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bd6592c37bb2:0-0 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732486585339.8ca644d2cada6dc44b363b7383dcd4fe. as already flushing 2024-11-24T22:16:43,287 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bd6592c37bb2:0-0 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732486585339.8ca644d2cada6dc44b363b7383dcd4fe. 2024-11-24T22:16:43,287 ERROR [RS_FLUSH_OPERATIONS-regionserver/bd6592c37bb2:0-0 {event_type=RS_FLUSH_REGIONS, pid=163}] handler.RSProcedureHandler(58): pid=163 java.io.IOException: Unable to complete flush {ENCODED => 8ca644d2cada6dc44b363b7383dcd4fe, NAME => 'TestAcidGuarantees,,1732486585339.8ca644d2cada6dc44b363b7383dcd4fe.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T22:16:43,288 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bd6592c37bb2:0-0 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=163 java.io.IOException: Unable to complete flush {ENCODED => 8ca644d2cada6dc44b363b7383dcd4fe, NAME => 'TestAcidGuarantees,,1732486585339.8ca644d2cada6dc44b363b7383dcd4fe.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T22:16:43,288 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37701 {}] master.HMaster(4177): Remote procedure failed, pid=163 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 8ca644d2cada6dc44b363b7383dcd4fe, NAME => 'TestAcidGuarantees,,1732486585339.8ca644d2cada6dc44b363b7383dcd4fe.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 8ca644d2cada6dc44b363b7383dcd4fe, NAME => 'TestAcidGuarantees,,1732486585339.8ca644d2cada6dc44b363b7383dcd4fe.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T22:16:43,307 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/bd6592c37bb2:0-9 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=312, cellSize=14282, totalHeapSize=32688, min timestamp=1732486603291, max timestamp=1732486603303Num uniques 1012; 2024-11-24T22:16:43,307 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/bd6592c37bb2:0-6 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=312, cellSize=14282, totalHeapSize=32688, min timestamp=1732486603291, max timestamp=1732486603303Num uniques 1012; 2024-11-24T22:16:43,307 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/bd6592c37bb2:0-9 {}] regionserver.CompactionPipeline(163): Suffix data size=46342, new segment data size=14282, suffix heap size=133216,new segment heap size=32688 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1012, new segment cells count=312 2024-11-24T22:16:43,307 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/bd6592c37bb2:0-6 {}] regionserver.CompactionPipeline(163): Suffix data size=46342, new segment data size=14282, suffix heap size=133216,new segment heap size=32688 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1012, new segment cells count=312 2024-11-24T22:16:43,308 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/bd6592c37bb2:0-7 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=312, cellSize=14282, totalHeapSize=32688, min timestamp=1732486603291, max timestamp=1732486603303Num uniques 1012; 2024-11-24T22:16:43,308 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/bd6592c37bb2:0-7 {}] regionserver.CompactionPipeline(163): Suffix data size=46342, new segment data size=14282, suffix heap size=133216,new segment heap size=32688 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1012, new segment cells count=312 2024-11-24T22:16:43,336 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/bd6592c37bb2:0-1 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=271, cellSize=12406, totalHeapSize=28424, min timestamp=1732486603321, max timestamp=1732486603330Num uniques 1171; 2024-11-24T22:16:43,336 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/bd6592c37bb2:0-1 {}] regionserver.CompactionPipeline(163): Suffix data size=53626, new segment data size=12406, suffix heap size=149752,new segment heap size=28424 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1171, new segment cells count=271 2024-11-24T22:16:43,336 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/bd6592c37bb2:0-4 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=271, cellSize=12406, totalHeapSize=28424, min timestamp=1732486603321, max timestamp=1732486603330Num uniques 1171; 2024-11-24T22:16:43,336 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/bd6592c37bb2:0-4 {}] regionserver.CompactionPipeline(163): Suffix data size=53626, new segment data size=12406, suffix heap size=149752,new segment heap size=28424 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1171, new segment cells count=271 2024-11-24T22:16:43,337 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/bd6592c37bb2:0-5 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=271, cellSize=12406, totalHeapSize=28424, min timestamp=1732486603321, max timestamp=1732486603330Num uniques 1171; 2024-11-24T22:16:43,337 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/bd6592c37bb2:0-5 {}] regionserver.CompactionPipeline(163): Suffix data size=53626, new segment data size=12406, suffix heap size=149752,new segment heap size=28424 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1171, new segment cells count=271 2024-11-24T22:16:43,351 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37701 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=162 2024-11-24T22:16:43,364 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40087 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8ca644d2cada6dc44b363b7383dcd4fe, server=bd6592c37bb2,40087,1732486423713 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-24T22:16:43,364 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40087 {}] ipc.CallRunner(138): callId: 80 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49060 deadline: 1732486663358, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8ca644d2cada6dc44b363b7383dcd4fe, server=bd6592c37bb2,40087,1732486423713 2024-11-24T22:16:43,365 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40087 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8ca644d2cada6dc44b363b7383dcd4fe, server=bd6592c37bb2,40087,1732486423713 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-24T22:16:43,365 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732486585339.8ca644d2cada6dc44b363b7383dcd4fe., hostname=bd6592c37bb2,40087,1732486423713, seqNum=5 , the old value is region=TestAcidGuarantees,,1732486585339.8ca644d2cada6dc44b363b7383dcd4fe., hostname=bd6592c37bb2,40087,1732486423713, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8ca644d2cada6dc44b363b7383dcd4fe, server=bd6592c37bb2,40087,1732486423713 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T22:16:43,365 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732486585339.8ca644d2cada6dc44b363b7383dcd4fe., hostname=bd6592c37bb2,40087,1732486423713, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8ca644d2cada6dc44b363b7383dcd4fe, server=bd6592c37bb2,40087,1732486423713 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T22:16:43,365 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40087 {}] ipc.CallRunner(138): callId: 167 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48986 deadline: 1732486663358, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8ca644d2cada6dc44b363b7383dcd4fe, server=bd6592c37bb2,40087,1732486423713 2024-11-24T22:16:43,365 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732486585339.8ca644d2cada6dc44b363b7383dcd4fe., hostname=bd6592c37bb2,40087,1732486423713, seqNum=5 because the exception is null or not the one we care about 2024-11-24T22:16:43,365 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732486585339.8ca644d2cada6dc44b363b7383dcd4fe., hostname=bd6592c37bb2,40087,1732486423713, seqNum=5 , the old value is region=TestAcidGuarantees,,1732486585339.8ca644d2cada6dc44b363b7383dcd4fe., hostname=bd6592c37bb2,40087,1732486423713, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8ca644d2cada6dc44b363b7383dcd4fe, server=bd6592c37bb2,40087,1732486423713 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T22:16:43,365 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732486585339.8ca644d2cada6dc44b363b7383dcd4fe., hostname=bd6592c37bb2,40087,1732486423713, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8ca644d2cada6dc44b363b7383dcd4fe, server=bd6592c37bb2,40087,1732486423713 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T22:16:43,365 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732486585339.8ca644d2cada6dc44b363b7383dcd4fe., hostname=bd6592c37bb2,40087,1732486423713, seqNum=5 because the exception is null or not the one we care about 2024-11-24T22:16:43,368 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/bd6592c37bb2:0-8 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=230, cellSize=10530, totalHeapSize=24160, min timestamp=1732486603356, max timestamp=1732486603363Num uniques 1130; 2024-11-24T22:16:43,368 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/bd6592c37bb2:0-8 {}] regionserver.CompactionPipeline(163): Suffix data size=51750, new segment data size=10530, suffix heap size=145488,new segment heap size=24160 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1130, new segment cells count=230 2024-11-24T22:16:43,368 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/bd6592c37bb2:0-0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=230, cellSize=10530, totalHeapSize=24160, min timestamp=1732486603356, max timestamp=1732486603363Num uniques 1130; 2024-11-24T22:16:43,368 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/bd6592c37bb2:0-0 {}] regionserver.CompactionPipeline(163): Suffix data size=51750, new segment data size=10530, suffix heap size=145488,new segment heap size=24160 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1130, new segment cells count=230 2024-11-24T22:16:43,369 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/bd6592c37bb2:0-2 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=230, cellSize=10530, totalHeapSize=24160, min timestamp=1732486603356, max timestamp=1732486603363Num uniques 1130; 2024-11-24T22:16:43,369 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/bd6592c37bb2:0-2 {}] regionserver.CompactionPipeline(163): Suffix data size=51750, new segment data size=10530, suffix heap size=145488,new segment heap size=24160 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1130, new segment cells count=230 2024-11-24T22:16:43,437 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/bd6592c37bb2:0-3 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=289, cellSize=13234, totalHeapSize=30296, min timestamp=1732486603370, max timestamp=1732486603433Num uniques 1089; 2024-11-24T22:16:43,437 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/bd6592c37bb2:0-3 {}] regionserver.CompactionPipeline(163): Suffix data size=49874, new segment data size=13234, suffix heap size=141224,new segment heap size=30296 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1089, new segment cells count=289 2024-11-24T22:16:43,437 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/bd6592c37bb2:0-9 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=289, cellSize=13234, totalHeapSize=30296, min timestamp=1732486603370, max timestamp=1732486603433Num uniques 1089; 2024-11-24T22:16:43,437 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/bd6592c37bb2:0-9 {}] regionserver.CompactionPipeline(163): Suffix data size=49874, new segment data size=13234, suffix heap size=141224,new segment heap size=30296 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1089, new segment cells count=289 2024-11-24T22:16:43,438 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/bd6592c37bb2:0-6 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=289, cellSize=13234, totalHeapSize=30296, min timestamp=1732486603370, max timestamp=1732486603433Num uniques 1089; 2024-11-24T22:16:43,438 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/bd6592c37bb2:0-6 {}] regionserver.CompactionPipeline(163): Suffix data size=49874, new segment data size=13234, suffix heap size=141224,new segment heap size=30296 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1089, new segment cells count=289 2024-11-24T22:16:43,440 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=40087 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=163 2024-11-24T22:16:43,440 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bd6592c37bb2:0-1 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732486585339.8ca644d2cada6dc44b363b7383dcd4fe. 2024-11-24T22:16:43,440 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bd6592c37bb2:0-1 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732486585339.8ca644d2cada6dc44b363b7383dcd4fe. as already flushing 2024-11-24T22:16:43,440 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bd6592c37bb2:0-1 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732486585339.8ca644d2cada6dc44b363b7383dcd4fe. 2024-11-24T22:16:43,440 ERROR [RS_FLUSH_OPERATIONS-regionserver/bd6592c37bb2:0-1 {event_type=RS_FLUSH_REGIONS, pid=163}] handler.RSProcedureHandler(58): pid=163 java.io.IOException: Unable to complete flush {ENCODED => 8ca644d2cada6dc44b363b7383dcd4fe, NAME => 'TestAcidGuarantees,,1732486585339.8ca644d2cada6dc44b363b7383dcd4fe.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T22:16:43,440 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bd6592c37bb2:0-1 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=163 java.io.IOException: Unable to complete flush {ENCODED => 8ca644d2cada6dc44b363b7383dcd4fe, NAME => 'TestAcidGuarantees,,1732486585339.8ca644d2cada6dc44b363b7383dcd4fe.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T22:16:43,441 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37701 {}] master.HMaster(4177): Remote procedure failed, pid=163 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 8ca644d2cada6dc44b363b7383dcd4fe, NAME => 'TestAcidGuarantees,,1732486585339.8ca644d2cada6dc44b363b7383dcd4fe.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 8ca644d2cada6dc44b363b7383dcd4fe, NAME => 'TestAcidGuarantees,,1732486585339.8ca644d2cada6dc44b363b7383dcd4fe.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T22:16:43,473 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40087 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8ca644d2cada6dc44b363b7383dcd4fe, server=bd6592c37bb2,40087,1732486423713 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-24T22:16:43,474 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40087 {}] ipc.CallRunner(138): callId: 99 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49026 deadline: 1732486663467, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8ca644d2cada6dc44b363b7383dcd4fe, server=bd6592c37bb2,40087,1732486423713 2024-11-24T22:16:43,474 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732486585339.8ca644d2cada6dc44b363b7383dcd4fe., hostname=bd6592c37bb2,40087,1732486423713, seqNum=5 , the old value is region=TestAcidGuarantees,,1732486585339.8ca644d2cada6dc44b363b7383dcd4fe., hostname=bd6592c37bb2,40087,1732486423713, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8ca644d2cada6dc44b363b7383dcd4fe, server=bd6592c37bb2,40087,1732486423713 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T22:16:43,474 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732486585339.8ca644d2cada6dc44b363b7383dcd4fe., hostname=bd6592c37bb2,40087,1732486423713, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8ca644d2cada6dc44b363b7383dcd4fe, server=bd6592c37bb2,40087,1732486423713 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T22:16:43,474 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732486585339.8ca644d2cada6dc44b363b7383dcd4fe., hostname=bd6592c37bb2,40087,1732486423713, seqNum=5 because the exception is null or not the one we care about 2024-11-24T22:16:43,477 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/bd6592c37bb2:0-1 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=198, cellSize=9068, totalHeapSize=20832, min timestamp=1732486603457, max timestamp=1732486603472Num uniques 1148; 2024-11-24T22:16:43,477 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/bd6592c37bb2:0-7 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=198, cellSize=9068, totalHeapSize=20832, min timestamp=1732486603457, max timestamp=1732486603472Num uniques 1148; 2024-11-24T22:16:43,477 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/bd6592c37bb2:0-1 {}] regionserver.CompactionPipeline(163): Suffix data size=52578, new segment data size=9068, suffix heap size=147360,new segment heap size=20832 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1148, new segment cells count=198 2024-11-24T22:16:43,477 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/bd6592c37bb2:0-7 {}] regionserver.CompactionPipeline(163): Suffix data size=52578, new segment data size=9068, suffix heap size=147360,new segment heap size=20832 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1148, new segment cells count=198 2024-11-24T22:16:43,477 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/bd6592c37bb2:0-4 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=198, cellSize=9068, totalHeapSize=20832, min timestamp=1732486603457, max timestamp=1732486603472Num uniques 1148; 2024-11-24T22:16:43,477 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/bd6592c37bb2:0-4 {}] regionserver.CompactionPipeline(163): Suffix data size=52578, new segment data size=9068, suffix heap size=147360,new segment heap size=20832 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1148, new segment cells count=198 2024-11-24T22:16:43,506 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/bd6592c37bb2:0-5 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=207, cellSize=9475, totalHeapSize=21768, min timestamp=1732486603490, max timestamp=1732486603502Num uniques 1057; 2024-11-24T22:16:43,506 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/bd6592c37bb2:0-5 {}] regionserver.CompactionPipeline(163): Suffix data size=48405, new segment data size=9475, suffix heap size=137896,new segment heap size=21768 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1057, new segment cells count=207 2024-11-24T22:16:43,506 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/bd6592c37bb2:0-8 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=207, cellSize=9475, totalHeapSize=21768, min timestamp=1732486603490, max timestamp=1732486603502Num uniques 1057; 2024-11-24T22:16:43,506 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/bd6592c37bb2:0-8 {}] regionserver.CompactionPipeline(163): Suffix data size=48405, new segment data size=9475, suffix heap size=137896,new segment heap size=21768 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1057, new segment cells count=207 2024-11-24T22:16:43,507 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/bd6592c37bb2:0-0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=207, cellSize=9475, totalHeapSize=21768, min timestamp=1732486603490, max timestamp=1732486603502Num uniques 1057; 2024-11-24T22:16:43,507 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/bd6592c37bb2:0-0 {}] regionserver.CompactionPipeline(163): Suffix data size=48405, new segment data size=9475, suffix heap size=137896,new segment heap size=21768 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1057, new segment cells count=207 2024-11-24T22:16:43,538 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/bd6592c37bb2:0-3 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=216, cellSize=9886, totalHeapSize=22704, min timestamp=1732486603523, max timestamp=1732486603534Num uniques 1066; 2024-11-24T22:16:43,538 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/bd6592c37bb2:0-3 {}] regionserver.CompactionPipeline(163): Suffix data size=48816, new segment data size=9886, suffix heap size=138832,new segment heap size=22704 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1066, new segment cells count=216 2024-11-24T22:16:43,538 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/bd6592c37bb2:0-2 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=216, cellSize=9886, totalHeapSize=22704, min timestamp=1732486603523, max timestamp=1732486603534Num uniques 1066; 2024-11-24T22:16:43,539 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/bd6592c37bb2:0-2 {}] regionserver.CompactionPipeline(163): Suffix data size=48816, new segment data size=9886, suffix heap size=138832,new segment heap size=22704 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1066, new segment cells count=216 2024-11-24T22:16:43,539 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/bd6592c37bb2:0-9 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=216, cellSize=9886, totalHeapSize=22704, min timestamp=1732486603523, max timestamp=1732486603534Num uniques 1066; 2024-11-24T22:16:43,539 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/bd6592c37bb2:0-9 {}] regionserver.CompactionPipeline(163): Suffix data size=48816, new segment data size=9886, suffix heap size=138832,new segment heap size=22704 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1066, new segment cells count=216 2024-11-24T22:16:43,570 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/bd6592c37bb2:0-1 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=225, cellSize=10300, totalHeapSize=23640, min timestamp=1732486603557, max timestamp=1732486603565Num uniques 1075; 2024-11-24T22:16:43,570 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/bd6592c37bb2:0-1 {}] regionserver.CompactionPipeline(163): Suffix data size=49230, new segment data size=10300, suffix heap size=139768,new segment heap size=23640 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1075, new segment cells count=225 2024-11-24T22:16:43,570 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/bd6592c37bb2:0-6 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=225, cellSize=10300, totalHeapSize=23640, min timestamp=1732486603557, max timestamp=1732486603565Num uniques 1075; 2024-11-24T22:16:43,570 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/bd6592c37bb2:0-6 {}] regionserver.CompactionPipeline(163): Suffix data size=49230, new segment data size=10300, suffix heap size=139768,new segment heap size=23640 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1075, new segment cells count=225 2024-11-24T22:16:43,570 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/bd6592c37bb2:0-7 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=225, cellSize=10300, totalHeapSize=23640, min timestamp=1732486603557, max timestamp=1732486603565Num uniques 1075; 2024-11-24T22:16:43,570 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/bd6592c37bb2:0-7 {}] regionserver.CompactionPipeline(163): Suffix data size=49230, new segment data size=10300, suffix heap size=139768,new segment heap size=23640 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1075, new segment cells count=225 2024-11-24T22:16:43,593 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=40087 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=163 2024-11-24T22:16:43,593 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bd6592c37bb2:0-2 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732486585339.8ca644d2cada6dc44b363b7383dcd4fe. 2024-11-24T22:16:43,593 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bd6592c37bb2:0-2 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732486585339.8ca644d2cada6dc44b363b7383dcd4fe. as already flushing 2024-11-24T22:16:43,593 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bd6592c37bb2:0-2 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732486585339.8ca644d2cada6dc44b363b7383dcd4fe. 2024-11-24T22:16:43,593 ERROR [RS_FLUSH_OPERATIONS-regionserver/bd6592c37bb2:0-2 {event_type=RS_FLUSH_REGIONS, pid=163}] handler.RSProcedureHandler(58): pid=163 java.io.IOException: Unable to complete flush {ENCODED => 8ca644d2cada6dc44b363b7383dcd4fe, NAME => 'TestAcidGuarantees,,1732486585339.8ca644d2cada6dc44b363b7383dcd4fe.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T22:16:43,594 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bd6592c37bb2:0-2 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=163 java.io.IOException: Unable to complete flush {ENCODED => 8ca644d2cada6dc44b363b7383dcd4fe, NAME => 'TestAcidGuarantees,,1732486585339.8ca644d2cada6dc44b363b7383dcd4fe.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T22:16:43,594 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37701 {}] master.HMaster(4177): Remote procedure failed, pid=163 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 8ca644d2cada6dc44b363b7383dcd4fe, NAME => 'TestAcidGuarantees,,1732486585339.8ca644d2cada6dc44b363b7383dcd4fe.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 8ca644d2cada6dc44b363b7383dcd4fe, NAME => 'TestAcidGuarantees,,1732486585339.8ca644d2cada6dc44b363b7383dcd4fe.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T22:16:43,597 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/bd6592c37bb2:0-4 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=284, cellSize=13004, totalHeapSize=29776, min timestamp=1732486603586, max timestamp=1732486603593Num uniques 1084; 2024-11-24T22:16:43,597 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/bd6592c37bb2:0-4 {}] regionserver.CompactionPipeline(163): Suffix data size=49644, new segment data size=13004, suffix heap size=140704,new segment heap size=29776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1084, new segment cells count=284 2024-11-24T22:16:43,597 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/bd6592c37bb2:0-5 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=284, cellSize=13004, totalHeapSize=29776, min timestamp=1732486603586, max timestamp=1732486603593Num uniques 1084; 2024-11-24T22:16:43,597 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/bd6592c37bb2:0-5 {}] regionserver.CompactionPipeline(163): Suffix data size=49644, new segment data size=13004, suffix heap size=140704,new segment heap size=29776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1084, new segment cells count=284 2024-11-24T22:16:43,600 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/bd6592c37bb2:0-4 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=284, cellSize=13004, totalHeapSize=29776, min timestamp=1732486603586, max timestamp=1732486603593Num uniques 1084; 2024-11-24T22:16:43,600 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/bd6592c37bb2:0-4 {}] regionserver.CompactionPipeline(163): Suffix data size=49644, new segment data size=13004, suffix heap size=140704,new segment heap size=29776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1084, new segment cells count=284 2024-11-24T22:16:43,617 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=11.18 KB at sequenceid=225 (bloomFilter=true), to=hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/8ca644d2cada6dc44b363b7383dcd4fe/.tmp/C/c86f7cc086c0485b914cc3b9e5711865 2024-11-24T22:16:43,621 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/8ca644d2cada6dc44b363b7383dcd4fe/.tmp/A/53ee59bfb6b54d9f8799663736d3c46a as hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/8ca644d2cada6dc44b363b7383dcd4fe/A/53ee59bfb6b54d9f8799663736d3c46a 2024-11-24T22:16:43,623 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40087 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8ca644d2cada6dc44b363b7383dcd4fe, server=bd6592c37bb2,40087,1732486423713 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-24T22:16:43,623 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40087 {}] ipc.CallRunner(138): callId: 188 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48986 deadline: 1732486663615, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8ca644d2cada6dc44b363b7383dcd4fe, server=bd6592c37bb2,40087,1732486423713 2024-11-24T22:16:43,624 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732486585339.8ca644d2cada6dc44b363b7383dcd4fe., hostname=bd6592c37bb2,40087,1732486423713, seqNum=5 , the old value is region=TestAcidGuarantees,,1732486585339.8ca644d2cada6dc44b363b7383dcd4fe., hostname=bd6592c37bb2,40087,1732486423713, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8ca644d2cada6dc44b363b7383dcd4fe, server=bd6592c37bb2,40087,1732486423713 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T22:16:43,624 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732486585339.8ca644d2cada6dc44b363b7383dcd4fe., hostname=bd6592c37bb2,40087,1732486423713, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8ca644d2cada6dc44b363b7383dcd4fe, server=bd6592c37bb2,40087,1732486423713 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T22:16:43,624 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732486585339.8ca644d2cada6dc44b363b7383dcd4fe., hostname=bd6592c37bb2,40087,1732486423713, seqNum=5 because the exception is null or not the one we care about 2024-11-24T22:16:43,624 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40087 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8ca644d2cada6dc44b363b7383dcd4fe, server=bd6592c37bb2,40087,1732486423713 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-24T22:16:43,624 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40087 {}] ipc.CallRunner(138): callId: 180 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49002 deadline: 1732486663618, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8ca644d2cada6dc44b363b7383dcd4fe, server=bd6592c37bb2,40087,1732486423713 2024-11-24T22:16:43,625 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40087 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8ca644d2cada6dc44b363b7383dcd4fe, server=bd6592c37bb2,40087,1732486423713 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-24T22:16:43,625 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40087 {}] ipc.CallRunner(138): callId: 105 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49026 deadline: 1732486663619, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8ca644d2cada6dc44b363b7383dcd4fe, server=bd6592c37bb2,40087,1732486423713 2024-11-24T22:16:43,625 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40087 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8ca644d2cada6dc44b363b7383dcd4fe, server=bd6592c37bb2,40087,1732486423713 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-24T22:16:43,625 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732486585339.8ca644d2cada6dc44b363b7383dcd4fe., hostname=bd6592c37bb2,40087,1732486423713, seqNum=5 , the old value is region=TestAcidGuarantees,,1732486585339.8ca644d2cada6dc44b363b7383dcd4fe., hostname=bd6592c37bb2,40087,1732486423713, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8ca644d2cada6dc44b363b7383dcd4fe, server=bd6592c37bb2,40087,1732486423713 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T22:16:43,626 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732486585339.8ca644d2cada6dc44b363b7383dcd4fe., hostname=bd6592c37bb2,40087,1732486423713, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8ca644d2cada6dc44b363b7383dcd4fe, server=bd6592c37bb2,40087,1732486423713 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T22:16:43,626 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40087 {}] ipc.CallRunner(138): callId: 205 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49050 deadline: 1732486663619, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8ca644d2cada6dc44b363b7383dcd4fe, server=bd6592c37bb2,40087,1732486423713 2024-11-24T22:16:43,626 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732486585339.8ca644d2cada6dc44b363b7383dcd4fe., hostname=bd6592c37bb2,40087,1732486423713, seqNum=5 because the exception is null or not the one we care about 2024-11-24T22:16:43,626 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732486585339.8ca644d2cada6dc44b363b7383dcd4fe., hostname=bd6592c37bb2,40087,1732486423713, seqNum=5 , the old value is region=TestAcidGuarantees,,1732486585339.8ca644d2cada6dc44b363b7383dcd4fe., hostname=bd6592c37bb2,40087,1732486423713, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8ca644d2cada6dc44b363b7383dcd4fe, server=bd6592c37bb2,40087,1732486423713 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T22:16:43,626 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732486585339.8ca644d2cada6dc44b363b7383dcd4fe., hostname=bd6592c37bb2,40087,1732486423713, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8ca644d2cada6dc44b363b7383dcd4fe, server=bd6592c37bb2,40087,1732486423713 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T22:16:43,626 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732486585339.8ca644d2cada6dc44b363b7383dcd4fe., hostname=bd6592c37bb2,40087,1732486423713, seqNum=5 because the exception is null or not the one we care about 2024-11-24T22:16:43,627 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/bd6592c37bb2:0-3 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=243, cellSize=11128, totalHeapSize=25512, min timestamp=1732486603614, max timestamp=1732486603622Num uniques 1143; 2024-11-24T22:16:43,627 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/bd6592c37bb2:0-3 {}] regionserver.CompactionPipeline(163): Suffix data size=52348, new segment data size=11128, suffix heap size=146840,new segment heap size=25512 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1143, new segment cells count=243 2024-11-24T22:16:43,627 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/bd6592c37bb2:0-0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=243, cellSize=11128, totalHeapSize=25512, min timestamp=1732486603614, max timestamp=1732486603622Num uniques 1143; 2024-11-24T22:16:43,627 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/bd6592c37bb2:0-0 {}] regionserver.CompactionPipeline(163): Suffix data size=52348, new segment data size=11128, suffix heap size=146840,new segment heap size=25512 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1143, new segment cells count=243 2024-11-24T22:16:43,627 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/bd6592c37bb2:0-2 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=243, cellSize=11128, totalHeapSize=25512, min timestamp=1732486603614, max timestamp=1732486603622Num uniques 1143; 2024-11-24T22:16:43,627 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/bd6592c37bb2:0-2 {}] regionserver.CompactionPipeline(163): Suffix data size=52348, new segment data size=11128, suffix heap size=146840,new segment heap size=25512 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1143, new segment cells count=243 2024-11-24T22:16:43,627 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732486585339.8ca644d2cada6dc44b363b7383dcd4fe., hostname=bd6592c37bb2,40087,1732486423713, seqNum=5 , the old value is region=TestAcidGuarantees,,1732486585339.8ca644d2cada6dc44b363b7383dcd4fe., hostname=bd6592c37bb2,40087,1732486423713, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8ca644d2cada6dc44b363b7383dcd4fe, server=bd6592c37bb2,40087,1732486423713 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T22:16:43,627 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732486585339.8ca644d2cada6dc44b363b7383dcd4fe., hostname=bd6592c37bb2,40087,1732486423713, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8ca644d2cada6dc44b363b7383dcd4fe, server=bd6592c37bb2,40087,1732486423713 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T22:16:43,627 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732486585339.8ca644d2cada6dc44b363b7383dcd4fe., hostname=bd6592c37bb2,40087,1732486423713, seqNum=5 because the exception is null or not the one we care about 2024-11-24T22:16:43,628 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/8ca644d2cada6dc44b363b7383dcd4fe/A/53ee59bfb6b54d9f8799663736d3c46a, entries=150, sequenceid=225, filesize=30.4 K 2024-11-24T22:16:43,628 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/8ca644d2cada6dc44b363b7383dcd4fe/.tmp/B/63e6053a21424b89a81ca99ff0a18861 as hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/8ca644d2cada6dc44b363b7383dcd4fe/B/63e6053a21424b89a81ca99ff0a18861 2024-11-24T22:16:43,631 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/8ca644d2cada6dc44b363b7383dcd4fe/B/63e6053a21424b89a81ca99ff0a18861, entries=150, sequenceid=225, filesize=11.9 K 2024-11-24T22:16:43,632 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/8ca644d2cada6dc44b363b7383dcd4fe/.tmp/C/c86f7cc086c0485b914cc3b9e5711865 as hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/8ca644d2cada6dc44b363b7383dcd4fe/C/c86f7cc086c0485b914cc3b9e5711865 2024-11-24T22:16:43,635 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/8ca644d2cada6dc44b363b7383dcd4fe/C/c86f7cc086c0485b914cc3b9e5711865, entries=150, sequenceid=225, filesize=11.9 K 2024-11-24T22:16:43,636 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~33.54 KB/34350, heapSize ~81.42 KB/83376, currentSize=40.25 KB/41220 for 8ca644d2cada6dc44b363b7383dcd4fe in 1301ms, sequenceid=225, compaction requested=true 2024-11-24T22:16:43,636 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 8ca644d2cada6dc44b363b7383dcd4fe: 2024-11-24T22:16:43,636 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 8ca644d2cada6dc44b363b7383dcd4fe:A, priority=-2147483648, current under compaction store size is 1 2024-11-24T22:16:43,636 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-24T22:16:43,636 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 8ca644d2cada6dc44b363b7383dcd4fe:B, priority=-2147483648, current under compaction store size is 2 2024-11-24T22:16:43,636 DEBUG [RS:0;bd6592c37bb2:40087-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-24T22:16:43,636 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-24T22:16:43,636 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 8ca644d2cada6dc44b363b7383dcd4fe:C, priority=-2147483648, current under compaction store size is 3 2024-11-24T22:16:43,636 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=2), splitQueue=0 2024-11-24T22:16:43,636 DEBUG [RS:0;bd6592c37bb2:40087-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-24T22:16:43,637 DEBUG [RS:0;bd6592c37bb2:40087-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36863 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-24T22:16:43,637 DEBUG [RS:0;bd6592c37bb2:40087-longCompactions-0 {}] regionserver.HStore(1541): 8ca644d2cada6dc44b363b7383dcd4fe/B is initiating minor compaction (all files) 2024-11-24T22:16:43,637 INFO [RS:0;bd6592c37bb2:40087-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 8ca644d2cada6dc44b363b7383dcd4fe/B in TestAcidGuarantees,,1732486585339.8ca644d2cada6dc44b363b7383dcd4fe. 2024-11-24T22:16:43,637 INFO [RS:0;bd6592c37bb2:40087-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/8ca644d2cada6dc44b363b7383dcd4fe/B/730efb19372148f8bdb95e42b451eb85, hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/8ca644d2cada6dc44b363b7383dcd4fe/B/0a9cb8108ab84a25a32039517a52cbeb, hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/8ca644d2cada6dc44b363b7383dcd4fe/B/63e6053a21424b89a81ca99ff0a18861] into tmpdir=hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/8ca644d2cada6dc44b363b7383dcd4fe/.tmp, totalSize=36.0 K 2024-11-24T22:16:43,637 DEBUG [RS:0;bd6592c37bb2:40087-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 93725 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-24T22:16:43,637 DEBUG [RS:0;bd6592c37bb2:40087-shortCompactions-0 {}] regionserver.HStore(1541): 8ca644d2cada6dc44b363b7383dcd4fe/A is initiating minor compaction (all files) 2024-11-24T22:16:43,637 INFO [RS:0;bd6592c37bb2:40087-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 8ca644d2cada6dc44b363b7383dcd4fe/A in TestAcidGuarantees,,1732486585339.8ca644d2cada6dc44b363b7383dcd4fe. 2024-11-24T22:16:43,637 INFO [RS:0;bd6592c37bb2:40087-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/8ca644d2cada6dc44b363b7383dcd4fe/A/09f6f98a4dde46df9e1923bd989b27de, hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/8ca644d2cada6dc44b363b7383dcd4fe/A/b8d8d281b8a04eeb8945f0979f39bfcc, hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/8ca644d2cada6dc44b363b7383dcd4fe/A/53ee59bfb6b54d9f8799663736d3c46a] into tmpdir=hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/8ca644d2cada6dc44b363b7383dcd4fe/.tmp, totalSize=91.5 K 2024-11-24T22:16:43,637 INFO [RS:0;bd6592c37bb2:40087-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=13 throughput controller=DefaultCompactionThroughputController [maxThroughput=57.69 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1732486585339.8ca644d2cada6dc44b363b7383dcd4fe. 2024-11-24T22:16:43,637 DEBUG [RS:0;bd6592c37bb2:40087-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1732486585339.8ca644d2cada6dc44b363b7383dcd4fe. files: [hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/8ca644d2cada6dc44b363b7383dcd4fe/A/09f6f98a4dde46df9e1923bd989b27de, hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/8ca644d2cada6dc44b363b7383dcd4fe/A/b8d8d281b8a04eeb8945f0979f39bfcc, hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/8ca644d2cada6dc44b363b7383dcd4fe/A/53ee59bfb6b54d9f8799663736d3c46a] 2024-11-24T22:16:43,638 DEBUG [RS:0;bd6592c37bb2:40087-longCompactions-0 {}] compactions.Compactor(225): Compacting 730efb19372148f8bdb95e42b451eb85, keycount=150, bloomtype=ROW, size=12.3 K, encoding=NONE, compression=NONE, seqNum=187, earliestPutTs=1732486600150 2024-11-24T22:16:43,638 DEBUG [RS:0;bd6592c37bb2:40087-shortCompactions-0 {}] compactions.Compactor(225): Compacting 09f6f98a4dde46df9e1923bd989b27de, keycount=150, bloomtype=ROW, size=30.8 K, encoding=NONE, compression=NONE, seqNum=187, earliestPutTs=1732486600150 2024-11-24T22:16:43,638 DEBUG [RS:0;bd6592c37bb2:40087-longCompactions-0 {}] compactions.Compactor(225): Compacting 0a9cb8108ab84a25a32039517a52cbeb, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=201, earliestPutTs=1732486600824 2024-11-24T22:16:43,638 DEBUG [RS:0;bd6592c37bb2:40087-shortCompactions-0 {}] compactions.Compactor(225): Compacting b8d8d281b8a04eeb8945f0979f39bfcc, keycount=150, bloomtype=ROW, size=30.4 K, encoding=NONE, compression=NONE, seqNum=201, earliestPutTs=1732486600824 2024-11-24T22:16:43,638 DEBUG [RS:0;bd6592c37bb2:40087-longCompactions-0 {}] compactions.Compactor(225): Compacting 63e6053a21424b89a81ca99ff0a18861, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=225, earliestPutTs=1732486601981 2024-11-24T22:16:43,638 DEBUG [RS:0;bd6592c37bb2:40087-shortCompactions-0 {}] compactions.Compactor(225): Compacting 53ee59bfb6b54d9f8799663736d3c46a, keycount=150, bloomtype=ROW, size=30.4 K, encoding=NONE, compression=NONE, seqNum=225, earliestPutTs=1732486601981 2024-11-24T22:16:43,640 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40087 {}] regionserver.HRegion(8855): Flush requested on 8ca644d2cada6dc44b363b7383dcd4fe 2024-11-24T22:16:43,641 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 8ca644d2cada6dc44b363b7383dcd4fe 3/3 column families, dataSize=53.67 KB heapSize=138.05 KB 2024-11-24T22:16:43,641 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 8ca644d2cada6dc44b363b7383dcd4fe, store=A 2024-11-24T22:16:43,641 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-24T22:16:43,641 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 8ca644d2cada6dc44b363b7383dcd4fe, store=B 2024-11-24T22:16:43,641 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-24T22:16:43,641 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 8ca644d2cada6dc44b363b7383dcd4fe, store=C 2024-11-24T22:16:43,641 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-24T22:16:43,646 INFO [RS:0;bd6592c37bb2:40087-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=8ca644d2cada6dc44b363b7383dcd4fe] 2024-11-24T22:16:43,647 INFO [RS:0;bd6592c37bb2:40087-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 8ca644d2cada6dc44b363b7383dcd4fe#B#compaction#524 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 57.69 MB/second 2024-11-24T22:16:43,648 DEBUG [RS:0;bd6592c37bb2:40087-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e20241124546f51e3583844bf9dca48d03edd3702_8ca644d2cada6dc44b363b7383dcd4fe store=[table=TestAcidGuarantees family=A region=8ca644d2cada6dc44b363b7383dcd4fe] 2024-11-24T22:16:43,648 DEBUG [RS:0;bd6592c37bb2:40087-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/8ca644d2cada6dc44b363b7383dcd4fe/.tmp/B/dc23ac5e77d74dc0906774d8a726505d is 50, key is test_row_0/B:col10/1732486602334/Put/seqid=0 2024-11-24T22:16:43,650 DEBUG [RS:0;bd6592c37bb2:40087-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e20241124546f51e3583844bf9dca48d03edd3702_8ca644d2cada6dc44b363b7383dcd4fe, store=[table=TestAcidGuarantees family=A region=8ca644d2cada6dc44b363b7383dcd4fe] 2024-11-24T22:16:43,650 DEBUG [RS:0;bd6592c37bb2:40087-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241124546f51e3583844bf9dca48d03edd3702_8ca644d2cada6dc44b363b7383dcd4fe because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=8ca644d2cada6dc44b363b7383dcd4fe] 2024-11-24T22:16:43,653 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411249e1c5c735c0242908fd028413e7a39dc_8ca644d2cada6dc44b363b7383dcd4fe is 50, key is test_row_0/A:col10/1732486603639/Put/seqid=0 2024-11-24T22:16:43,661 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45929 is added to blk_1073742446_1622 (size=12663) 2024-11-24T22:16:43,665 DEBUG [RS:0;bd6592c37bb2:40087-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/8ca644d2cada6dc44b363b7383dcd4fe/.tmp/B/dc23ac5e77d74dc0906774d8a726505d as hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/8ca644d2cada6dc44b363b7383dcd4fe/B/dc23ac5e77d74dc0906774d8a726505d 2024-11-24T22:16:43,669 INFO [RS:0;bd6592c37bb2:40087-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 8ca644d2cada6dc44b363b7383dcd4fe/B of 8ca644d2cada6dc44b363b7383dcd4fe into dc23ac5e77d74dc0906774d8a726505d(size=12.4 K), total size for store is 12.4 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-24T22:16:43,669 DEBUG [RS:0;bd6592c37bb2:40087-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 8ca644d2cada6dc44b363b7383dcd4fe: 2024-11-24T22:16:43,669 INFO [RS:0;bd6592c37bb2:40087-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732486585339.8ca644d2cada6dc44b363b7383dcd4fe., storeName=8ca644d2cada6dc44b363b7383dcd4fe/B, priority=13, startTime=1732486603636; duration=0sec 2024-11-24T22:16:43,669 DEBUG [RS:0;bd6592c37bb2:40087-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-24T22:16:43,669 DEBUG [RS:0;bd6592c37bb2:40087-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 8ca644d2cada6dc44b363b7383dcd4fe:B 2024-11-24T22:16:43,669 DEBUG [RS:0;bd6592c37bb2:40087-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-24T22:16:43,669 DEBUG [RS:0;bd6592c37bb2:40087-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36863 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-24T22:16:43,669 DEBUG [RS:0;bd6592c37bb2:40087-longCompactions-0 {}] regionserver.HStore(1541): 8ca644d2cada6dc44b363b7383dcd4fe/C is initiating minor compaction (all files) 2024-11-24T22:16:43,669 INFO [RS:0;bd6592c37bb2:40087-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 8ca644d2cada6dc44b363b7383dcd4fe/C in TestAcidGuarantees,,1732486585339.8ca644d2cada6dc44b363b7383dcd4fe. 2024-11-24T22:16:43,669 INFO [RS:0;bd6592c37bb2:40087-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/8ca644d2cada6dc44b363b7383dcd4fe/C/09a368f67227449abf941025f6c40c9f, hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/8ca644d2cada6dc44b363b7383dcd4fe/C/b6da2cc25fa44492ad282381b9a45c04, hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/8ca644d2cada6dc44b363b7383dcd4fe/C/c86f7cc086c0485b914cc3b9e5711865] into tmpdir=hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/8ca644d2cada6dc44b363b7383dcd4fe/.tmp, totalSize=36.0 K 2024-11-24T22:16:43,670 DEBUG [RS:0;bd6592c37bb2:40087-longCompactions-0 {}] compactions.Compactor(225): Compacting 09a368f67227449abf941025f6c40c9f, keycount=150, bloomtype=ROW, size=12.3 K, encoding=NONE, compression=NONE, seqNum=187, earliestPutTs=1732486600150 2024-11-24T22:16:43,670 DEBUG [RS:0;bd6592c37bb2:40087-longCompactions-0 {}] compactions.Compactor(225): Compacting b6da2cc25fa44492ad282381b9a45c04, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=201, earliestPutTs=1732486600824 2024-11-24T22:16:43,670 DEBUG [RS:0;bd6592c37bb2:40087-longCompactions-0 {}] compactions.Compactor(225): Compacting c86f7cc086c0485b914cc3b9e5711865, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=225, earliestPutTs=1732486601981 2024-11-24T22:16:43,674 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45929 is added to blk_1073742447_1623 (size=4469) 2024-11-24T22:16:43,675 INFO [RS:0;bd6592c37bb2:40087-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 8ca644d2cada6dc44b363b7383dcd4fe#A#compaction#525 average throughput is 0.84 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 57.69 MB/second 2024-11-24T22:16:43,676 DEBUG [RS:0;bd6592c37bb2:40087-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/8ca644d2cada6dc44b363b7383dcd4fe/.tmp/A/7263a115caa04d7d9b7c84d0a6c9cce4 is 175, key is test_row_0/A:col10/1732486602334/Put/seqid=0 2024-11-24T22:16:43,686 INFO [RS:0;bd6592c37bb2:40087-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 8ca644d2cada6dc44b363b7383dcd4fe#C#compaction#527 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 57.69 MB/second 2024-11-24T22:16:43,686 DEBUG [RS:0;bd6592c37bb2:40087-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/8ca644d2cada6dc44b363b7383dcd4fe/.tmp/C/ef06905113514929b387959c8588fb32 is 50, key is test_row_0/C:col10/1732486602334/Put/seqid=0 2024-11-24T22:16:43,693 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45929 is added to blk_1073742448_1624 (size=12454) 2024-11-24T22:16:43,693 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T22:16:43,694 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45929 is added to blk_1073742449_1625 (size=31617) 2024-11-24T22:16:43,696 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45929 is added to blk_1073742450_1626 (size=12663) 2024-11-24T22:16:43,697 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411249e1c5c735c0242908fd028413e7a39dc_8ca644d2cada6dc44b363b7383dcd4fe to hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411249e1c5c735c0242908fd028413e7a39dc_8ca644d2cada6dc44b363b7383dcd4fe 2024-11-24T22:16:43,698 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/8ca644d2cada6dc44b363b7383dcd4fe/.tmp/A/99464ebe699642cfa110755eb79a0020, store: [table=TestAcidGuarantees family=A region=8ca644d2cada6dc44b363b7383dcd4fe] 2024-11-24T22:16:43,698 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/8ca644d2cada6dc44b363b7383dcd4fe/.tmp/A/99464ebe699642cfa110755eb79a0020 is 175, key is test_row_0/A:col10/1732486603639/Put/seqid=0 2024-11-24T22:16:43,701 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45929 is added to blk_1073742451_1627 (size=31255) 2024-11-24T22:16:43,740 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/bd6592c37bb2:0-1 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=259, cellSize=11855, totalHeapSize=27176, min timestamp=1732486603717, max timestamp=1732486603737Num uniques 859; 2024-11-24T22:16:43,740 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/bd6592c37bb2:0-1 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=11855, suffix heap size=117064,new segment heap size=27176 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=259 2024-11-24T22:16:43,740 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/bd6592c37bb2:0-6 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=259, cellSize=11855, totalHeapSize=27176, min timestamp=1732486603717, max timestamp=1732486603737Num uniques 859; 2024-11-24T22:16:43,740 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/bd6592c37bb2:0-6 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=11855, suffix heap size=117064,new segment heap size=27176 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=259 2024-11-24T22:16:43,740 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/bd6592c37bb2:0-9 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=259, cellSize=11855, totalHeapSize=27176, min timestamp=1732486603717, max timestamp=1732486603737Num uniques 859; 2024-11-24T22:16:43,740 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/bd6592c37bb2:0-9 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=11855, suffix heap size=117064,new segment heap size=27176 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=259 2024-11-24T22:16:43,746 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=40087 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=163 2024-11-24T22:16:43,746 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bd6592c37bb2:0-0 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732486585339.8ca644d2cada6dc44b363b7383dcd4fe. 2024-11-24T22:16:43,746 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bd6592c37bb2:0-0 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732486585339.8ca644d2cada6dc44b363b7383dcd4fe. as already flushing 2024-11-24T22:16:43,746 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bd6592c37bb2:0-0 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732486585339.8ca644d2cada6dc44b363b7383dcd4fe. 2024-11-24T22:16:43,746 ERROR [RS_FLUSH_OPERATIONS-regionserver/bd6592c37bb2:0-0 {event_type=RS_FLUSH_REGIONS, pid=163}] handler.RSProcedureHandler(58): pid=163 java.io.IOException: Unable to complete flush {ENCODED => 8ca644d2cada6dc44b363b7383dcd4fe, NAME => 'TestAcidGuarantees,,1732486585339.8ca644d2cada6dc44b363b7383dcd4fe.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T22:16:43,746 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bd6592c37bb2:0-0 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=163 java.io.IOException: Unable to complete flush {ENCODED => 8ca644d2cada6dc44b363b7383dcd4fe, NAME => 'TestAcidGuarantees,,1732486585339.8ca644d2cada6dc44b363b7383dcd4fe.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T22:16:43,747 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37701 {}] master.HMaster(4177): Remote procedure failed, pid=163 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 8ca644d2cada6dc44b363b7383dcd4fe, NAME => 'TestAcidGuarantees,,1732486585339.8ca644d2cada6dc44b363b7383dcd4fe.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 8ca644d2cada6dc44b363b7383dcd4fe, NAME => 'TestAcidGuarantees,,1732486585339.8ca644d2cada6dc44b363b7383dcd4fe.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T22:16:43,763 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40087 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8ca644d2cada6dc44b363b7383dcd4fe, server=bd6592c37bb2,40087,1732486423713 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-24T22:16:43,764 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40087 {}] ipc.CallRunner(138): callId: 109 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49026 deadline: 1732486663756, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8ca644d2cada6dc44b363b7383dcd4fe, server=bd6592c37bb2,40087,1732486423713 2024-11-24T22:16:43,764 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732486585339.8ca644d2cada6dc44b363b7383dcd4fe., hostname=bd6592c37bb2,40087,1732486423713, seqNum=5 , the old value is region=TestAcidGuarantees,,1732486585339.8ca644d2cada6dc44b363b7383dcd4fe., hostname=bd6592c37bb2,40087,1732486423713, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8ca644d2cada6dc44b363b7383dcd4fe, server=bd6592c37bb2,40087,1732486423713 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T22:16:43,764 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40087 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8ca644d2cada6dc44b363b7383dcd4fe, server=bd6592c37bb2,40087,1732486423713 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-24T22:16:43,764 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732486585339.8ca644d2cada6dc44b363b7383dcd4fe., hostname=bd6592c37bb2,40087,1732486423713, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8ca644d2cada6dc44b363b7383dcd4fe, server=bd6592c37bb2,40087,1732486423713 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T22:16:43,764 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732486585339.8ca644d2cada6dc44b363b7383dcd4fe., hostname=bd6592c37bb2,40087,1732486423713, seqNum=5 because the exception is null or not the one we care about 2024-11-24T22:16:43,764 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40087 {}] ipc.CallRunner(138): callId: 209 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49050 deadline: 1732486663757, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8ca644d2cada6dc44b363b7383dcd4fe, server=bd6592c37bb2,40087,1732486423713 2024-11-24T22:16:43,764 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732486585339.8ca644d2cada6dc44b363b7383dcd4fe., hostname=bd6592c37bb2,40087,1732486423713, seqNum=5 , the old value is region=TestAcidGuarantees,,1732486585339.8ca644d2cada6dc44b363b7383dcd4fe., hostname=bd6592c37bb2,40087,1732486423713, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8ca644d2cada6dc44b363b7383dcd4fe, server=bd6592c37bb2,40087,1732486423713 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T22:16:43,764 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40087 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8ca644d2cada6dc44b363b7383dcd4fe, server=bd6592c37bb2,40087,1732486423713 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-24T22:16:43,764 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732486585339.8ca644d2cada6dc44b363b7383dcd4fe., hostname=bd6592c37bb2,40087,1732486423713, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8ca644d2cada6dc44b363b7383dcd4fe, server=bd6592c37bb2,40087,1732486423713 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T22:16:43,764 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732486585339.8ca644d2cada6dc44b363b7383dcd4fe., hostname=bd6592c37bb2,40087,1732486423713, seqNum=5 because the exception is null or not the one we care about 2024-11-24T22:16:43,764 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40087 {}] ipc.CallRunner(138): callId: 184 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49002 deadline: 1732486663758, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8ca644d2cada6dc44b363b7383dcd4fe, server=bd6592c37bb2,40087,1732486423713 2024-11-24T22:16:43,765 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732486585339.8ca644d2cada6dc44b363b7383dcd4fe., hostname=bd6592c37bb2,40087,1732486423713, seqNum=5 , the old value is region=TestAcidGuarantees,,1732486585339.8ca644d2cada6dc44b363b7383dcd4fe., hostname=bd6592c37bb2,40087,1732486423713, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8ca644d2cada6dc44b363b7383dcd4fe, server=bd6592c37bb2,40087,1732486423713 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T22:16:43,765 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732486585339.8ca644d2cada6dc44b363b7383dcd4fe., hostname=bd6592c37bb2,40087,1732486423713, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8ca644d2cada6dc44b363b7383dcd4fe, server=bd6592c37bb2,40087,1732486423713 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T22:16:43,765 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732486585339.8ca644d2cada6dc44b363b7383dcd4fe., hostname=bd6592c37bb2,40087,1732486423713, seqNum=5 because the exception is null or not the one we care about 2024-11-24T22:16:43,766 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40087 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8ca644d2cada6dc44b363b7383dcd4fe, server=bd6592c37bb2,40087,1732486423713 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-24T22:16:43,766 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40087 {}] ipc.CallRunner(138): callId: 193 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48986 deadline: 1732486663760, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8ca644d2cada6dc44b363b7383dcd4fe, server=bd6592c37bb2,40087,1732486423713 2024-11-24T22:16:43,766 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732486585339.8ca644d2cada6dc44b363b7383dcd4fe., hostname=bd6592c37bb2,40087,1732486423713, seqNum=5 , the old value is region=TestAcidGuarantees,,1732486585339.8ca644d2cada6dc44b363b7383dcd4fe., hostname=bd6592c37bb2,40087,1732486423713, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8ca644d2cada6dc44b363b7383dcd4fe, server=bd6592c37bb2,40087,1732486423713 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T22:16:43,766 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732486585339.8ca644d2cada6dc44b363b7383dcd4fe., hostname=bd6592c37bb2,40087,1732486423713, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8ca644d2cada6dc44b363b7383dcd4fe, server=bd6592c37bb2,40087,1732486423713 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T22:16:43,766 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732486585339.8ca644d2cada6dc44b363b7383dcd4fe., hostname=bd6592c37bb2,40087,1732486423713, seqNum=5 because the exception is null or not the one we care about 2024-11-24T22:16:43,768 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40087 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8ca644d2cada6dc44b363b7383dcd4fe, server=bd6592c37bb2,40087,1732486423713 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-24T22:16:43,768 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40087 {}] ipc.CallRunner(138): callId: 124 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49060 deadline: 1732486663763, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8ca644d2cada6dc44b363b7383dcd4fe, server=bd6592c37bb2,40087,1732486423713 2024-11-24T22:16:43,768 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732486585339.8ca644d2cada6dc44b363b7383dcd4fe., hostname=bd6592c37bb2,40087,1732486423713, seqNum=5 , the old value is region=TestAcidGuarantees,,1732486585339.8ca644d2cada6dc44b363b7383dcd4fe., hostname=bd6592c37bb2,40087,1732486423713, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8ca644d2cada6dc44b363b7383dcd4fe, server=bd6592c37bb2,40087,1732486423713 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T22:16:43,768 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732486585339.8ca644d2cada6dc44b363b7383dcd4fe., hostname=bd6592c37bb2,40087,1732486423713, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8ca644d2cada6dc44b363b7383dcd4fe, server=bd6592c37bb2,40087,1732486423713 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T22:16:43,768 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732486585339.8ca644d2cada6dc44b363b7383dcd4fe., hostname=bd6592c37bb2,40087,1732486423713, seqNum=5 because the exception is null or not the one we care about 2024-11-24T22:16:43,874 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40087 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8ca644d2cada6dc44b363b7383dcd4fe, server=bd6592c37bb2,40087,1732486423713 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-24T22:16:43,874 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40087 {}] ipc.CallRunner(138): callId: 110 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49026 deadline: 1732486663871, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8ca644d2cada6dc44b363b7383dcd4fe, server=bd6592c37bb2,40087,1732486423713 2024-11-24T22:16:43,874 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40087 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8ca644d2cada6dc44b363b7383dcd4fe, server=bd6592c37bb2,40087,1732486423713 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-24T22:16:43,874 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732486585339.8ca644d2cada6dc44b363b7383dcd4fe., hostname=bd6592c37bb2,40087,1732486423713, seqNum=5 , the old value is region=TestAcidGuarantees,,1732486585339.8ca644d2cada6dc44b363b7383dcd4fe., hostname=bd6592c37bb2,40087,1732486423713, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8ca644d2cada6dc44b363b7383dcd4fe, server=bd6592c37bb2,40087,1732486423713 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T22:16:43,874 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40087 {}] ipc.CallRunner(138): callId: 210 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49050 deadline: 1732486663871, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8ca644d2cada6dc44b363b7383dcd4fe, server=bd6592c37bb2,40087,1732486423713 2024-11-24T22:16:43,874 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732486585339.8ca644d2cada6dc44b363b7383dcd4fe., hostname=bd6592c37bb2,40087,1732486423713, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8ca644d2cada6dc44b363b7383dcd4fe, server=bd6592c37bb2,40087,1732486423713 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T22:16:43,874 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732486585339.8ca644d2cada6dc44b363b7383dcd4fe., hostname=bd6592c37bb2,40087,1732486423713, seqNum=5 because the exception is null or not the one we care about 2024-11-24T22:16:43,874 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40087 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8ca644d2cada6dc44b363b7383dcd4fe, server=bd6592c37bb2,40087,1732486423713 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-24T22:16:43,874 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732486585339.8ca644d2cada6dc44b363b7383dcd4fe., hostname=bd6592c37bb2,40087,1732486423713, seqNum=5 , the old value is region=TestAcidGuarantees,,1732486585339.8ca644d2cada6dc44b363b7383dcd4fe., hostname=bd6592c37bb2,40087,1732486423713, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8ca644d2cada6dc44b363b7383dcd4fe, server=bd6592c37bb2,40087,1732486423713 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T22:16:43,874 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732486585339.8ca644d2cada6dc44b363b7383dcd4fe., hostname=bd6592c37bb2,40087,1732486423713, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8ca644d2cada6dc44b363b7383dcd4fe, server=bd6592c37bb2,40087,1732486423713 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T22:16:43,874 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732486585339.8ca644d2cada6dc44b363b7383dcd4fe., hostname=bd6592c37bb2,40087,1732486423713, seqNum=5 because the exception is null or not the one we care about 2024-11-24T22:16:43,874 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40087 {}] ipc.CallRunner(138): callId: 185 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49002 deadline: 1732486663871, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8ca644d2cada6dc44b363b7383dcd4fe, server=bd6592c37bb2,40087,1732486423713 2024-11-24T22:16:43,875 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732486585339.8ca644d2cada6dc44b363b7383dcd4fe., hostname=bd6592c37bb2,40087,1732486423713, seqNum=5 , the old value is region=TestAcidGuarantees,,1732486585339.8ca644d2cada6dc44b363b7383dcd4fe., hostname=bd6592c37bb2,40087,1732486423713, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8ca644d2cada6dc44b363b7383dcd4fe, server=bd6592c37bb2,40087,1732486423713 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T22:16:43,875 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40087 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8ca644d2cada6dc44b363b7383dcd4fe, server=bd6592c37bb2,40087,1732486423713 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-24T22:16:43,875 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732486585339.8ca644d2cada6dc44b363b7383dcd4fe., hostname=bd6592c37bb2,40087,1732486423713, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8ca644d2cada6dc44b363b7383dcd4fe, server=bd6592c37bb2,40087,1732486423713 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T22:16:43,875 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732486585339.8ca644d2cada6dc44b363b7383dcd4fe., hostname=bd6592c37bb2,40087,1732486423713, seqNum=5 because the exception is null or not the one we care about 2024-11-24T22:16:43,875 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40087 {}] ipc.CallRunner(138): callId: 194 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48986 deadline: 1732486663871, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8ca644d2cada6dc44b363b7383dcd4fe, server=bd6592c37bb2,40087,1732486423713 2024-11-24T22:16:43,875 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732486585339.8ca644d2cada6dc44b363b7383dcd4fe., hostname=bd6592c37bb2,40087,1732486423713, seqNum=5 , the old value is region=TestAcidGuarantees,,1732486585339.8ca644d2cada6dc44b363b7383dcd4fe., hostname=bd6592c37bb2,40087,1732486423713, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8ca644d2cada6dc44b363b7383dcd4fe, server=bd6592c37bb2,40087,1732486423713 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T22:16:43,875 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40087 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8ca644d2cada6dc44b363b7383dcd4fe, server=bd6592c37bb2,40087,1732486423713 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-24T22:16:43,875 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732486585339.8ca644d2cada6dc44b363b7383dcd4fe., hostname=bd6592c37bb2,40087,1732486423713, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8ca644d2cada6dc44b363b7383dcd4fe, server=bd6592c37bb2,40087,1732486423713 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T22:16:43,875 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732486585339.8ca644d2cada6dc44b363b7383dcd4fe., hostname=bd6592c37bb2,40087,1732486423713, seqNum=5 because the exception is null or not the one we care about 2024-11-24T22:16:43,875 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40087 {}] ipc.CallRunner(138): callId: 125 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49060 deadline: 1732486663871, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8ca644d2cada6dc44b363b7383dcd4fe, server=bd6592c37bb2,40087,1732486423713 2024-11-24T22:16:43,875 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732486585339.8ca644d2cada6dc44b363b7383dcd4fe., hostname=bd6592c37bb2,40087,1732486423713, seqNum=5 , the old value is region=TestAcidGuarantees,,1732486585339.8ca644d2cada6dc44b363b7383dcd4fe., hostname=bd6592c37bb2,40087,1732486423713, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8ca644d2cada6dc44b363b7383dcd4fe, server=bd6592c37bb2,40087,1732486423713 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T22:16:43,875 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732486585339.8ca644d2cada6dc44b363b7383dcd4fe., hostname=bd6592c37bb2,40087,1732486423713, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8ca644d2cada6dc44b363b7383dcd4fe, server=bd6592c37bb2,40087,1732486423713 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T22:16:43,875 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732486585339.8ca644d2cada6dc44b363b7383dcd4fe., hostname=bd6592c37bb2,40087,1732486423713, seqNum=5 because the exception is null or not the one we care about 2024-11-24T22:16:43,899 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=40087 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=163 2024-11-24T22:16:43,899 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bd6592c37bb2:0-1 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732486585339.8ca644d2cada6dc44b363b7383dcd4fe. 2024-11-24T22:16:43,899 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bd6592c37bb2:0-1 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732486585339.8ca644d2cada6dc44b363b7383dcd4fe. as already flushing 2024-11-24T22:16:43,899 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bd6592c37bb2:0-1 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732486585339.8ca644d2cada6dc44b363b7383dcd4fe. 2024-11-24T22:16:43,899 ERROR [RS_FLUSH_OPERATIONS-regionserver/bd6592c37bb2:0-1 {event_type=RS_FLUSH_REGIONS, pid=163}] handler.RSProcedureHandler(58): pid=163 java.io.IOException: Unable to complete flush {ENCODED => 8ca644d2cada6dc44b363b7383dcd4fe, NAME => 'TestAcidGuarantees,,1732486585339.8ca644d2cada6dc44b363b7383dcd4fe.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T22:16:43,899 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bd6592c37bb2:0-1 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=163 java.io.IOException: Unable to complete flush {ENCODED => 8ca644d2cada6dc44b363b7383dcd4fe, NAME => 'TestAcidGuarantees,,1732486585339.8ca644d2cada6dc44b363b7383dcd4fe.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T22:16:43,900 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37701 {}] master.HMaster(4177): Remote procedure failed, pid=163 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 8ca644d2cada6dc44b363b7383dcd4fe, NAME => 'TestAcidGuarantees,,1732486585339.8ca644d2cada6dc44b363b7383dcd4fe.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 8ca644d2cada6dc44b363b7383dcd4fe, NAME => 'TestAcidGuarantees,,1732486585339.8ca644d2cada6dc44b363b7383dcd4fe.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T22:16:44,053 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=40087 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=163 2024-11-24T22:16:44,054 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bd6592c37bb2:0-2 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732486585339.8ca644d2cada6dc44b363b7383dcd4fe. 2024-11-24T22:16:44,054 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bd6592c37bb2:0-2 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732486585339.8ca644d2cada6dc44b363b7383dcd4fe. as already flushing 2024-11-24T22:16:44,054 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bd6592c37bb2:0-2 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732486585339.8ca644d2cada6dc44b363b7383dcd4fe. 2024-11-24T22:16:44,054 ERROR [RS_FLUSH_OPERATIONS-regionserver/bd6592c37bb2:0-2 {event_type=RS_FLUSH_REGIONS, pid=163}] handler.RSProcedureHandler(58): pid=163 java.io.IOException: Unable to complete flush {ENCODED => 8ca644d2cada6dc44b363b7383dcd4fe, NAME => 'TestAcidGuarantees,,1732486585339.8ca644d2cada6dc44b363b7383dcd4fe.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T22:16:44,054 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bd6592c37bb2:0-2 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=163 java.io.IOException: Unable to complete flush {ENCODED => 8ca644d2cada6dc44b363b7383dcd4fe, NAME => 'TestAcidGuarantees,,1732486585339.8ca644d2cada6dc44b363b7383dcd4fe.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T22:16:44,055 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37701 {}] master.HMaster(4177): Remote procedure failed, pid=163 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 8ca644d2cada6dc44b363b7383dcd4fe, NAME => 'TestAcidGuarantees,,1732486585339.8ca644d2cada6dc44b363b7383dcd4fe.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 8ca644d2cada6dc44b363b7383dcd4fe, NAME => 'TestAcidGuarantees,,1732486585339.8ca644d2cada6dc44b363b7383dcd4fe.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T22:16:44,084 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40087 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8ca644d2cada6dc44b363b7383dcd4fe, server=bd6592c37bb2,40087,1732486423713 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-24T22:16:44,084 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40087 {}] ipc.CallRunner(138): callId: 111 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49026 deadline: 1732486664080, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8ca644d2cada6dc44b363b7383dcd4fe, server=bd6592c37bb2,40087,1732486423713 2024-11-24T22:16:44,084 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40087 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8ca644d2cada6dc44b363b7383dcd4fe, server=bd6592c37bb2,40087,1732486423713 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-24T22:16:44,084 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40087 {}] ipc.CallRunner(138): callId: 211 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49050 deadline: 1732486664080, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8ca644d2cada6dc44b363b7383dcd4fe, server=bd6592c37bb2,40087,1732486423713 2024-11-24T22:16:44,084 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732486585339.8ca644d2cada6dc44b363b7383dcd4fe., hostname=bd6592c37bb2,40087,1732486423713, seqNum=5 , the old value is region=TestAcidGuarantees,,1732486585339.8ca644d2cada6dc44b363b7383dcd4fe., hostname=bd6592c37bb2,40087,1732486423713, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8ca644d2cada6dc44b363b7383dcd4fe, server=bd6592c37bb2,40087,1732486423713 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T22:16:44,084 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732486585339.8ca644d2cada6dc44b363b7383dcd4fe., hostname=bd6592c37bb2,40087,1732486423713, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8ca644d2cada6dc44b363b7383dcd4fe, server=bd6592c37bb2,40087,1732486423713 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T22:16:44,085 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732486585339.8ca644d2cada6dc44b363b7383dcd4fe., hostname=bd6592c37bb2,40087,1732486423713, seqNum=5 because the exception is null or not the one we care about 2024-11-24T22:16:44,085 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732486585339.8ca644d2cada6dc44b363b7383dcd4fe., hostname=bd6592c37bb2,40087,1732486423713, seqNum=5 , the old value is region=TestAcidGuarantees,,1732486585339.8ca644d2cada6dc44b363b7383dcd4fe., hostname=bd6592c37bb2,40087,1732486423713, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8ca644d2cada6dc44b363b7383dcd4fe, server=bd6592c37bb2,40087,1732486423713 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T22:16:44,085 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40087 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8ca644d2cada6dc44b363b7383dcd4fe, server=bd6592c37bb2,40087,1732486423713 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-24T22:16:44,085 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732486585339.8ca644d2cada6dc44b363b7383dcd4fe., hostname=bd6592c37bb2,40087,1732486423713, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8ca644d2cada6dc44b363b7383dcd4fe, server=bd6592c37bb2,40087,1732486423713 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T22:16:44,085 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732486585339.8ca644d2cada6dc44b363b7383dcd4fe., hostname=bd6592c37bb2,40087,1732486423713, seqNum=5 because the exception is null or not the one we care about 2024-11-24T22:16:44,085 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40087 {}] ipc.CallRunner(138): callId: 186 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49002 deadline: 1732486664081, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8ca644d2cada6dc44b363b7383dcd4fe, server=bd6592c37bb2,40087,1732486423713 2024-11-24T22:16:44,085 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40087 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8ca644d2cada6dc44b363b7383dcd4fe, server=bd6592c37bb2,40087,1732486423713 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-24T22:16:44,085 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732486585339.8ca644d2cada6dc44b363b7383dcd4fe., hostname=bd6592c37bb2,40087,1732486423713, seqNum=5 , the old value is region=TestAcidGuarantees,,1732486585339.8ca644d2cada6dc44b363b7383dcd4fe., hostname=bd6592c37bb2,40087,1732486423713, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8ca644d2cada6dc44b363b7383dcd4fe, server=bd6592c37bb2,40087,1732486423713 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T22:16:44,085 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732486585339.8ca644d2cada6dc44b363b7383dcd4fe., hostname=bd6592c37bb2,40087,1732486423713, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8ca644d2cada6dc44b363b7383dcd4fe, server=bd6592c37bb2,40087,1732486423713 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T22:16:44,085 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732486585339.8ca644d2cada6dc44b363b7383dcd4fe., hostname=bd6592c37bb2,40087,1732486423713, seqNum=5 because the exception is null or not the one we care about 2024-11-24T22:16:44,085 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40087 {}] ipc.CallRunner(138): callId: 195 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48986 deadline: 1732486664081, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8ca644d2cada6dc44b363b7383dcd4fe, server=bd6592c37bb2,40087,1732486423713 2024-11-24T22:16:44,085 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40087 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8ca644d2cada6dc44b363b7383dcd4fe, server=bd6592c37bb2,40087,1732486423713 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-24T22:16:44,085 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732486585339.8ca644d2cada6dc44b363b7383dcd4fe., hostname=bd6592c37bb2,40087,1732486423713, seqNum=5 , the old value is region=TestAcidGuarantees,,1732486585339.8ca644d2cada6dc44b363b7383dcd4fe., hostname=bd6592c37bb2,40087,1732486423713, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8ca644d2cada6dc44b363b7383dcd4fe, server=bd6592c37bb2,40087,1732486423713 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T22:16:44,085 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732486585339.8ca644d2cada6dc44b363b7383dcd4fe., hostname=bd6592c37bb2,40087,1732486423713, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8ca644d2cada6dc44b363b7383dcd4fe, server=bd6592c37bb2,40087,1732486423713 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T22:16:44,085 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40087 {}] ipc.CallRunner(138): callId: 126 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49060 deadline: 1732486664081, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8ca644d2cada6dc44b363b7383dcd4fe, server=bd6592c37bb2,40087,1732486423713 2024-11-24T22:16:44,085 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732486585339.8ca644d2cada6dc44b363b7383dcd4fe., hostname=bd6592c37bb2,40087,1732486423713, seqNum=5 because the exception is null or not the one we care about 2024-11-24T22:16:44,085 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732486585339.8ca644d2cada6dc44b363b7383dcd4fe., hostname=bd6592c37bb2,40087,1732486423713, seqNum=5 , the old value is region=TestAcidGuarantees,,1732486585339.8ca644d2cada6dc44b363b7383dcd4fe., hostname=bd6592c37bb2,40087,1732486423713, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8ca644d2cada6dc44b363b7383dcd4fe, server=bd6592c37bb2,40087,1732486423713 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T22:16:44,085 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732486585339.8ca644d2cada6dc44b363b7383dcd4fe., hostname=bd6592c37bb2,40087,1732486423713, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8ca644d2cada6dc44b363b7383dcd4fe, server=bd6592c37bb2,40087,1732486423713 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T22:16:44,085 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732486585339.8ca644d2cada6dc44b363b7383dcd4fe., hostname=bd6592c37bb2,40087,1732486423713, seqNum=5 because the exception is null or not the one we care about 2024-11-24T22:16:44,098 DEBUG [RS:0;bd6592c37bb2:40087-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/8ca644d2cada6dc44b363b7383dcd4fe/.tmp/A/7263a115caa04d7d9b7c84d0a6c9cce4 as hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/8ca644d2cada6dc44b363b7383dcd4fe/A/7263a115caa04d7d9b7c84d0a6c9cce4 2024-11-24T22:16:44,100 DEBUG [RS:0;bd6592c37bb2:40087-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/8ca644d2cada6dc44b363b7383dcd4fe/.tmp/C/ef06905113514929b387959c8588fb32 as hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/8ca644d2cada6dc44b363b7383dcd4fe/C/ef06905113514929b387959c8588fb32 2024-11-24T22:16:44,101 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=695, memsize=17.9 K, hasBloomFilter=true, into tmp file hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/8ca644d2cada6dc44b363b7383dcd4fe/.tmp/A/99464ebe699642cfa110755eb79a0020 2024-11-24T22:16:44,103 INFO [RS:0;bd6592c37bb2:40087-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 8ca644d2cada6dc44b363b7383dcd4fe/A of 8ca644d2cada6dc44b363b7383dcd4fe into 7263a115caa04d7d9b7c84d0a6c9cce4(size=30.9 K), total size for store is 30.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-24T22:16:44,103 DEBUG [RS:0;bd6592c37bb2:40087-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 8ca644d2cada6dc44b363b7383dcd4fe: 2024-11-24T22:16:44,103 INFO [RS:0;bd6592c37bb2:40087-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732486585339.8ca644d2cada6dc44b363b7383dcd4fe., storeName=8ca644d2cada6dc44b363b7383dcd4fe/A, priority=13, startTime=1732486603636; duration=0sec 2024-11-24T22:16:44,103 DEBUG [RS:0;bd6592c37bb2:40087-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-24T22:16:44,103 DEBUG [RS:0;bd6592c37bb2:40087-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 8ca644d2cada6dc44b363b7383dcd4fe:A 2024-11-24T22:16:44,104 INFO [RS:0;bd6592c37bb2:40087-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 8ca644d2cada6dc44b363b7383dcd4fe/C of 8ca644d2cada6dc44b363b7383dcd4fe into ef06905113514929b387959c8588fb32(size=12.4 K), total size for store is 12.4 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-24T22:16:44,104 DEBUG [RS:0;bd6592c37bb2:40087-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 8ca644d2cada6dc44b363b7383dcd4fe: 2024-11-24T22:16:44,104 INFO [RS:0;bd6592c37bb2:40087-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732486585339.8ca644d2cada6dc44b363b7383dcd4fe., storeName=8ca644d2cada6dc44b363b7383dcd4fe/C, priority=13, startTime=1732486603636; duration=0sec 2024-11-24T22:16:44,105 DEBUG [RS:0;bd6592c37bb2:40087-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-24T22:16:44,105 DEBUG [RS:0;bd6592c37bb2:40087-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 8ca644d2cada6dc44b363b7383dcd4fe:C 2024-11-24T22:16:44,108 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/8ca644d2cada6dc44b363b7383dcd4fe/.tmp/B/6b52569f4db642aebd605b9ea3871fa5 is 50, key is test_row_0/B:col10/1732486603639/Put/seqid=0 2024-11-24T22:16:44,112 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45929 is added to blk_1073742452_1628 (size=12301) 2024-11-24T22:16:44,113 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=695 (bloomFilter=true), to=hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/8ca644d2cada6dc44b363b7383dcd4fe/.tmp/B/6b52569f4db642aebd605b9ea3871fa5 2024-11-24T22:16:44,119 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/8ca644d2cada6dc44b363b7383dcd4fe/.tmp/C/943ba0f17f0c449e98e26c922ce03508 is 50, key is test_row_0/C:col10/1732486603639/Put/seqid=0 2024-11-24T22:16:44,122 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45929 is added to blk_1073742453_1629 (size=12301) 2024-11-24T22:16:44,206 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=40087 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=163 2024-11-24T22:16:44,206 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bd6592c37bb2:0-0 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732486585339.8ca644d2cada6dc44b363b7383dcd4fe. 2024-11-24T22:16:44,207 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bd6592c37bb2:0-0 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732486585339.8ca644d2cada6dc44b363b7383dcd4fe. as already flushing 2024-11-24T22:16:44,207 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bd6592c37bb2:0-0 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732486585339.8ca644d2cada6dc44b363b7383dcd4fe. 2024-11-24T22:16:44,207 ERROR [RS_FLUSH_OPERATIONS-regionserver/bd6592c37bb2:0-0 {event_type=RS_FLUSH_REGIONS, pid=163}] handler.RSProcedureHandler(58): pid=163 java.io.IOException: Unable to complete flush {ENCODED => 8ca644d2cada6dc44b363b7383dcd4fe, NAME => 'TestAcidGuarantees,,1732486585339.8ca644d2cada6dc44b363b7383dcd4fe.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T22:16:44,207 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bd6592c37bb2:0-0 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=163 java.io.IOException: Unable to complete flush {ENCODED => 8ca644d2cada6dc44b363b7383dcd4fe, NAME => 'TestAcidGuarantees,,1732486585339.8ca644d2cada6dc44b363b7383dcd4fe.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T22:16:44,207 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37701 {}] master.HMaster(4177): Remote procedure failed, pid=163 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 8ca644d2cada6dc44b363b7383dcd4fe, NAME => 'TestAcidGuarantees,,1732486585339.8ca644d2cada6dc44b363b7383dcd4fe.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 8ca644d2cada6dc44b363b7383dcd4fe, NAME => 'TestAcidGuarantees,,1732486585339.8ca644d2cada6dc44b363b7383dcd4fe.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T22:16:44,359 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=40087 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=163 2024-11-24T22:16:44,359 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bd6592c37bb2:0-1 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732486585339.8ca644d2cada6dc44b363b7383dcd4fe. 2024-11-24T22:16:44,359 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bd6592c37bb2:0-1 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732486585339.8ca644d2cada6dc44b363b7383dcd4fe. as already flushing 2024-11-24T22:16:44,359 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bd6592c37bb2:0-1 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732486585339.8ca644d2cada6dc44b363b7383dcd4fe. 2024-11-24T22:16:44,360 ERROR [RS_FLUSH_OPERATIONS-regionserver/bd6592c37bb2:0-1 {event_type=RS_FLUSH_REGIONS, pid=163}] handler.RSProcedureHandler(58): pid=163 java.io.IOException: Unable to complete flush {ENCODED => 8ca644d2cada6dc44b363b7383dcd4fe, NAME => 'TestAcidGuarantees,,1732486585339.8ca644d2cada6dc44b363b7383dcd4fe.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T22:16:44,360 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bd6592c37bb2:0-1 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=163 java.io.IOException: Unable to complete flush {ENCODED => 8ca644d2cada6dc44b363b7383dcd4fe, NAME => 'TestAcidGuarantees,,1732486585339.8ca644d2cada6dc44b363b7383dcd4fe.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T22:16:44,360 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37701 {}] master.HMaster(4177): Remote procedure failed, pid=163 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 8ca644d2cada6dc44b363b7383dcd4fe, NAME => 'TestAcidGuarantees,,1732486585339.8ca644d2cada6dc44b363b7383dcd4fe.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 8ca644d2cada6dc44b363b7383dcd4fe, NAME => 'TestAcidGuarantees,,1732486585339.8ca644d2cada6dc44b363b7383dcd4fe.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T22:16:44,361 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37701 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=162 2024-11-24T22:16:44,394 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40087 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8ca644d2cada6dc44b363b7383dcd4fe, server=bd6592c37bb2,40087,1732486423713 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-24T22:16:44,394 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40087 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8ca644d2cada6dc44b363b7383dcd4fe, server=bd6592c37bb2,40087,1732486423713 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-24T22:16:44,394 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40087 {}] ipc.CallRunner(138): callId: 112 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49026 deadline: 1732486664390, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8ca644d2cada6dc44b363b7383dcd4fe, server=bd6592c37bb2,40087,1732486423713 2024-11-24T22:16:44,395 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40087 {}] ipc.CallRunner(138): callId: 212 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49050 deadline: 1732486664390, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8ca644d2cada6dc44b363b7383dcd4fe, server=bd6592c37bb2,40087,1732486423713 2024-11-24T22:16:44,395 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732486585339.8ca644d2cada6dc44b363b7383dcd4fe., hostname=bd6592c37bb2,40087,1732486423713, seqNum=5 , the old value is region=TestAcidGuarantees,,1732486585339.8ca644d2cada6dc44b363b7383dcd4fe., hostname=bd6592c37bb2,40087,1732486423713, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8ca644d2cada6dc44b363b7383dcd4fe, server=bd6592c37bb2,40087,1732486423713 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T22:16:44,395 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40087 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8ca644d2cada6dc44b363b7383dcd4fe, server=bd6592c37bb2,40087,1732486423713 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-24T22:16:44,395 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732486585339.8ca644d2cada6dc44b363b7383dcd4fe., hostname=bd6592c37bb2,40087,1732486423713, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8ca644d2cada6dc44b363b7383dcd4fe, server=bd6592c37bb2,40087,1732486423713 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T22:16:44,395 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732486585339.8ca644d2cada6dc44b363b7383dcd4fe., hostname=bd6592c37bb2,40087,1732486423713, seqNum=5 because the exception is null or not the one we care about 2024-11-24T22:16:44,395 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40087 {}] ipc.CallRunner(138): callId: 187 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49002 deadline: 1732486664391, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8ca644d2cada6dc44b363b7383dcd4fe, server=bd6592c37bb2,40087,1732486423713 2024-11-24T22:16:44,395 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732486585339.8ca644d2cada6dc44b363b7383dcd4fe., hostname=bd6592c37bb2,40087,1732486423713, seqNum=5 , the old value is region=TestAcidGuarantees,,1732486585339.8ca644d2cada6dc44b363b7383dcd4fe., hostname=bd6592c37bb2,40087,1732486423713, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8ca644d2cada6dc44b363b7383dcd4fe, server=bd6592c37bb2,40087,1732486423713 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T22:16:44,395 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732486585339.8ca644d2cada6dc44b363b7383dcd4fe., hostname=bd6592c37bb2,40087,1732486423713, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8ca644d2cada6dc44b363b7383dcd4fe, server=bd6592c37bb2,40087,1732486423713 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T22:16:44,395 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732486585339.8ca644d2cada6dc44b363b7383dcd4fe., hostname=bd6592c37bb2,40087,1732486423713, seqNum=5 because the exception is null or not the one we care about 2024-11-24T22:16:44,395 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40087 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8ca644d2cada6dc44b363b7383dcd4fe, server=bd6592c37bb2,40087,1732486423713 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-24T22:16:44,395 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732486585339.8ca644d2cada6dc44b363b7383dcd4fe., hostname=bd6592c37bb2,40087,1732486423713, seqNum=5 , the old value is region=TestAcidGuarantees,,1732486585339.8ca644d2cada6dc44b363b7383dcd4fe., hostname=bd6592c37bb2,40087,1732486423713, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8ca644d2cada6dc44b363b7383dcd4fe, server=bd6592c37bb2,40087,1732486423713 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T22:16:44,395 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40087 {}] ipc.CallRunner(138): callId: 196 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48986 deadline: 1732486664391, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8ca644d2cada6dc44b363b7383dcd4fe, server=bd6592c37bb2,40087,1732486423713 2024-11-24T22:16:44,395 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732486585339.8ca644d2cada6dc44b363b7383dcd4fe., hostname=bd6592c37bb2,40087,1732486423713, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8ca644d2cada6dc44b363b7383dcd4fe, server=bd6592c37bb2,40087,1732486423713 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T22:16:44,395 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732486585339.8ca644d2cada6dc44b363b7383dcd4fe., hostname=bd6592c37bb2,40087,1732486423713, seqNum=5 because the exception is null or not the one we care about 2024-11-24T22:16:44,395 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732486585339.8ca644d2cada6dc44b363b7383dcd4fe., hostname=bd6592c37bb2,40087,1732486423713, seqNum=5 , the old value is region=TestAcidGuarantees,,1732486585339.8ca644d2cada6dc44b363b7383dcd4fe., hostname=bd6592c37bb2,40087,1732486423713, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8ca644d2cada6dc44b363b7383dcd4fe, server=bd6592c37bb2,40087,1732486423713 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T22:16:44,395 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40087 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8ca644d2cada6dc44b363b7383dcd4fe, server=bd6592c37bb2,40087,1732486423713 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-24T22:16:44,396 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732486585339.8ca644d2cada6dc44b363b7383dcd4fe., hostname=bd6592c37bb2,40087,1732486423713, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8ca644d2cada6dc44b363b7383dcd4fe, server=bd6592c37bb2,40087,1732486423713 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T22:16:44,396 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732486585339.8ca644d2cada6dc44b363b7383dcd4fe., hostname=bd6592c37bb2,40087,1732486423713, seqNum=5 because the exception is null or not the one we care about 2024-11-24T22:16:44,396 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40087 {}] ipc.CallRunner(138): callId: 127 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49060 deadline: 1732486664391, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8ca644d2cada6dc44b363b7383dcd4fe, server=bd6592c37bb2,40087,1732486423713 2024-11-24T22:16:44,396 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732486585339.8ca644d2cada6dc44b363b7383dcd4fe., hostname=bd6592c37bb2,40087,1732486423713, seqNum=5 , the old value is region=TestAcidGuarantees,,1732486585339.8ca644d2cada6dc44b363b7383dcd4fe., hostname=bd6592c37bb2,40087,1732486423713, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8ca644d2cada6dc44b363b7383dcd4fe, server=bd6592c37bb2,40087,1732486423713 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T22:16:44,396 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732486585339.8ca644d2cada6dc44b363b7383dcd4fe., hostname=bd6592c37bb2,40087,1732486423713, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8ca644d2cada6dc44b363b7383dcd4fe, server=bd6592c37bb2,40087,1732486423713 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T22:16:44,396 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732486585339.8ca644d2cada6dc44b363b7383dcd4fe., hostname=bd6592c37bb2,40087,1732486423713, seqNum=5 because the exception is null or not the one we care about 2024-11-24T22:16:44,512 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=40087 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=163 2024-11-24T22:16:44,512 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bd6592c37bb2:0-2 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732486585339.8ca644d2cada6dc44b363b7383dcd4fe. 2024-11-24T22:16:44,512 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bd6592c37bb2:0-2 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732486585339.8ca644d2cada6dc44b363b7383dcd4fe. as already flushing 2024-11-24T22:16:44,512 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bd6592c37bb2:0-2 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732486585339.8ca644d2cada6dc44b363b7383dcd4fe. 2024-11-24T22:16:44,512 ERROR [RS_FLUSH_OPERATIONS-regionserver/bd6592c37bb2:0-2 {event_type=RS_FLUSH_REGIONS, pid=163}] handler.RSProcedureHandler(58): pid=163 java.io.IOException: Unable to complete flush {ENCODED => 8ca644d2cada6dc44b363b7383dcd4fe, NAME => 'TestAcidGuarantees,,1732486585339.8ca644d2cada6dc44b363b7383dcd4fe.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T22:16:44,512 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bd6592c37bb2:0-2 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=163 java.io.IOException: Unable to complete flush {ENCODED => 8ca644d2cada6dc44b363b7383dcd4fe, NAME => 'TestAcidGuarantees,,1732486585339.8ca644d2cada6dc44b363b7383dcd4fe.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T22:16:44,513 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37701 {}] master.HMaster(4177): Remote procedure failed, pid=163 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 8ca644d2cada6dc44b363b7383dcd4fe, NAME => 'TestAcidGuarantees,,1732486585339.8ca644d2cada6dc44b363b7383dcd4fe.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 8ca644d2cada6dc44b363b7383dcd4fe, NAME => 'TestAcidGuarantees,,1732486585339.8ca644d2cada6dc44b363b7383dcd4fe.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T22:16:44,523 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=695 (bloomFilter=true), to=hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/8ca644d2cada6dc44b363b7383dcd4fe/.tmp/C/943ba0f17f0c449e98e26c922ce03508 2024-11-24T22:16:44,526 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/8ca644d2cada6dc44b363b7383dcd4fe/.tmp/A/99464ebe699642cfa110755eb79a0020 as hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/8ca644d2cada6dc44b363b7383dcd4fe/A/99464ebe699642cfa110755eb79a0020 2024-11-24T22:16:44,529 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/8ca644d2cada6dc44b363b7383dcd4fe/A/99464ebe699642cfa110755eb79a0020, entries=150, sequenceid=695, filesize=30.5 K 2024-11-24T22:16:44,530 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/8ca644d2cada6dc44b363b7383dcd4fe/.tmp/B/6b52569f4db642aebd605b9ea3871fa5 as hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/8ca644d2cada6dc44b363b7383dcd4fe/B/6b52569f4db642aebd605b9ea3871fa5 2024-11-24T22:16:44,533 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/8ca644d2cada6dc44b363b7383dcd4fe/B/6b52569f4db642aebd605b9ea3871fa5, entries=150, sequenceid=695, filesize=12.0 K 2024-11-24T22:16:44,534 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/8ca644d2cada6dc44b363b7383dcd4fe/.tmp/C/943ba0f17f0c449e98e26c922ce03508 as hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/8ca644d2cada6dc44b363b7383dcd4fe/C/943ba0f17f0c449e98e26c922ce03508 2024-11-24T22:16:44,536 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/8ca644d2cada6dc44b363b7383dcd4fe/C/943ba0f17f0c449e98e26c922ce03508, entries=150, sequenceid=695, filesize=12.0 K 2024-11-24T22:16:44,537 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~53.67 KB/54960, heapSize ~138 KB/141312, currentSize=134.18 KB/137400 for 8ca644d2cada6dc44b363b7383dcd4fe in 897ms, sequenceid=695, compaction requested=false 2024-11-24T22:16:44,537 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 8ca644d2cada6dc44b363b7383dcd4fe: 2024-11-24T22:16:44,665 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=40087 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=163 2024-11-24T22:16:44,665 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bd6592c37bb2:0-0 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732486585339.8ca644d2cada6dc44b363b7383dcd4fe. 2024-11-24T22:16:44,665 INFO [RS_FLUSH_OPERATIONS-regionserver/bd6592c37bb2:0-0 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.HRegion(2902): Flushing 8ca644d2cada6dc44b363b7383dcd4fe 3/3 column families, dataSize=134.18 KB heapSize=375.61 KB 2024-11-24T22:16:44,666 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bd6592c37bb2:0-0 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 8ca644d2cada6dc44b363b7383dcd4fe, store=A 2024-11-24T22:16:44,666 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bd6592c37bb2:0-0 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-24T22:16:44,666 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bd6592c37bb2:0-0 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 8ca644d2cada6dc44b363b7383dcd4fe, store=B 2024-11-24T22:16:44,666 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bd6592c37bb2:0-0 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-24T22:16:44,666 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bd6592c37bb2:0-0 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 8ca644d2cada6dc44b363b7383dcd4fe, store=C 2024-11-24T22:16:44,666 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bd6592c37bb2:0-0 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-24T22:16:44,689 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bd6592c37bb2:0-0 {event_type=RS_FLUSH_REGIONS, pid=163}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411248ec87d903bc94f8089f0ac1a67cd13d1_8ca644d2cada6dc44b363b7383dcd4fe is 50, key is test_row_0/A:col10/1732486603762/Put/seqid=0 2024-11-24T22:16:44,731 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45929 is added to blk_1073742454_1630 (size=12454) 2024-11-24T22:16:44,903 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40087 {}] regionserver.HRegion(8855): Flush requested on 8ca644d2cada6dc44b363b7383dcd4fe 2024-11-24T22:16:44,903 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732486585339.8ca644d2cada6dc44b363b7383dcd4fe. as already flushing 2024-11-24T22:16:44,914 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40087 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8ca644d2cada6dc44b363b7383dcd4fe, server=bd6592c37bb2,40087,1732486423713 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-24T22:16:44,915 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40087 {}] ipc.CallRunner(138): callId: 213 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49050 deadline: 1732486664910, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8ca644d2cada6dc44b363b7383dcd4fe, server=bd6592c37bb2,40087,1732486423713 2024-11-24T22:16:44,915 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40087 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8ca644d2cada6dc44b363b7383dcd4fe, server=bd6592c37bb2,40087,1732486423713 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-24T22:16:44,915 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732486585339.8ca644d2cada6dc44b363b7383dcd4fe., hostname=bd6592c37bb2,40087,1732486423713, seqNum=5 , the old value is region=TestAcidGuarantees,,1732486585339.8ca644d2cada6dc44b363b7383dcd4fe., hostname=bd6592c37bb2,40087,1732486423713, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8ca644d2cada6dc44b363b7383dcd4fe, server=bd6592c37bb2,40087,1732486423713 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T22:16:44,915 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732486585339.8ca644d2cada6dc44b363b7383dcd4fe., hostname=bd6592c37bb2,40087,1732486423713, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8ca644d2cada6dc44b363b7383dcd4fe, server=bd6592c37bb2,40087,1732486423713 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T22:16:44,915 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40087 {}] ipc.CallRunner(138): callId: 128 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49060 deadline: 1732486664910, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8ca644d2cada6dc44b363b7383dcd4fe, server=bd6592c37bb2,40087,1732486423713 2024-11-24T22:16:44,915 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732486585339.8ca644d2cada6dc44b363b7383dcd4fe., hostname=bd6592c37bb2,40087,1732486423713, seqNum=5 because the exception is null or not the one we care about 2024-11-24T22:16:44,915 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732486585339.8ca644d2cada6dc44b363b7383dcd4fe., hostname=bd6592c37bb2,40087,1732486423713, seqNum=5 , the old value is region=TestAcidGuarantees,,1732486585339.8ca644d2cada6dc44b363b7383dcd4fe., hostname=bd6592c37bb2,40087,1732486423713, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8ca644d2cada6dc44b363b7383dcd4fe, server=bd6592c37bb2,40087,1732486423713 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T22:16:44,915 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732486585339.8ca644d2cada6dc44b363b7383dcd4fe., hostname=bd6592c37bb2,40087,1732486423713, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8ca644d2cada6dc44b363b7383dcd4fe, server=bd6592c37bb2,40087,1732486423713 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T22:16:44,915 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732486585339.8ca644d2cada6dc44b363b7383dcd4fe., hostname=bd6592c37bb2,40087,1732486423713, seqNum=5 because the exception is null or not the one we care about 2024-11-24T22:16:44,918 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40087 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8ca644d2cada6dc44b363b7383dcd4fe, server=bd6592c37bb2,40087,1732486423713 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-24T22:16:44,918 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40087 {}] ipc.CallRunner(138): callId: 199 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48986 deadline: 1732486664911, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8ca644d2cada6dc44b363b7383dcd4fe, server=bd6592c37bb2,40087,1732486423713 2024-11-24T22:16:44,918 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732486585339.8ca644d2cada6dc44b363b7383dcd4fe., hostname=bd6592c37bb2,40087,1732486423713, seqNum=5 , the old value is region=TestAcidGuarantees,,1732486585339.8ca644d2cada6dc44b363b7383dcd4fe., hostname=bd6592c37bb2,40087,1732486423713, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8ca644d2cada6dc44b363b7383dcd4fe, server=bd6592c37bb2,40087,1732486423713 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T22:16:44,918 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40087 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8ca644d2cada6dc44b363b7383dcd4fe, server=bd6592c37bb2,40087,1732486423713 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-24T22:16:44,918 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732486585339.8ca644d2cada6dc44b363b7383dcd4fe., hostname=bd6592c37bb2,40087,1732486423713, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8ca644d2cada6dc44b363b7383dcd4fe, server=bd6592c37bb2,40087,1732486423713 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T22:16:44,918 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732486585339.8ca644d2cada6dc44b363b7383dcd4fe., hostname=bd6592c37bb2,40087,1732486423713, seqNum=5 because the exception is null or not the one we care about 2024-11-24T22:16:44,918 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40087 {}] ipc.CallRunner(138): callId: 190 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49002 deadline: 1732486664913, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8ca644d2cada6dc44b363b7383dcd4fe, server=bd6592c37bb2,40087,1732486423713 2024-11-24T22:16:44,919 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732486585339.8ca644d2cada6dc44b363b7383dcd4fe., hostname=bd6592c37bb2,40087,1732486423713, seqNum=5 , the old value is region=TestAcidGuarantees,,1732486585339.8ca644d2cada6dc44b363b7383dcd4fe., hostname=bd6592c37bb2,40087,1732486423713, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8ca644d2cada6dc44b363b7383dcd4fe, server=bd6592c37bb2,40087,1732486423713 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T22:16:44,919 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732486585339.8ca644d2cada6dc44b363b7383dcd4fe., hostname=bd6592c37bb2,40087,1732486423713, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8ca644d2cada6dc44b363b7383dcd4fe, server=bd6592c37bb2,40087,1732486423713 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T22:16:44,919 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732486585339.8ca644d2cada6dc44b363b7383dcd4fe., hostname=bd6592c37bb2,40087,1732486423713, seqNum=5 because the exception is null or not the one we care about 2024-11-24T22:16:44,919 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40087 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8ca644d2cada6dc44b363b7383dcd4fe, server=bd6592c37bb2,40087,1732486423713 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-24T22:16:44,919 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40087 {}] ipc.CallRunner(138): callId: 116 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49026 deadline: 1732486664915, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8ca644d2cada6dc44b363b7383dcd4fe, server=bd6592c37bb2,40087,1732486423713 2024-11-24T22:16:44,919 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732486585339.8ca644d2cada6dc44b363b7383dcd4fe., hostname=bd6592c37bb2,40087,1732486423713, seqNum=5 , the old value is region=TestAcidGuarantees,,1732486585339.8ca644d2cada6dc44b363b7383dcd4fe., hostname=bd6592c37bb2,40087,1732486423713, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8ca644d2cada6dc44b363b7383dcd4fe, server=bd6592c37bb2,40087,1732486423713 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T22:16:44,919 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732486585339.8ca644d2cada6dc44b363b7383dcd4fe., hostname=bd6592c37bb2,40087,1732486423713, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8ca644d2cada6dc44b363b7383dcd4fe, server=bd6592c37bb2,40087,1732486423713 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T22:16:44,920 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732486585339.8ca644d2cada6dc44b363b7383dcd4fe., hostname=bd6592c37bb2,40087,1732486423713, seqNum=5 because the exception is null or not the one we care about 2024-11-24T22:16:45,022 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40087 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8ca644d2cada6dc44b363b7383dcd4fe, server=bd6592c37bb2,40087,1732486423713 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-24T22:16:45,022 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40087 {}] ipc.CallRunner(138): callId: 200 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48986 deadline: 1732486665021, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8ca644d2cada6dc44b363b7383dcd4fe, server=bd6592c37bb2,40087,1732486423713 2024-11-24T22:16:45,023 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40087 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8ca644d2cada6dc44b363b7383dcd4fe, server=bd6592c37bb2,40087,1732486423713 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-24T22:16:45,023 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732486585339.8ca644d2cada6dc44b363b7383dcd4fe., hostname=bd6592c37bb2,40087,1732486423713, seqNum=5 , the old value is region=TestAcidGuarantees,,1732486585339.8ca644d2cada6dc44b363b7383dcd4fe., hostname=bd6592c37bb2,40087,1732486423713, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8ca644d2cada6dc44b363b7383dcd4fe, server=bd6592c37bb2,40087,1732486423713 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T22:16:45,023 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732486585339.8ca644d2cada6dc44b363b7383dcd4fe., hostname=bd6592c37bb2,40087,1732486423713, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8ca644d2cada6dc44b363b7383dcd4fe, server=bd6592c37bb2,40087,1732486423713 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T22:16:45,023 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40087 {}] ipc.CallRunner(138): callId: 117 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49026 deadline: 1732486665021, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8ca644d2cada6dc44b363b7383dcd4fe, server=bd6592c37bb2,40087,1732486423713 2024-11-24T22:16:45,023 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732486585339.8ca644d2cada6dc44b363b7383dcd4fe., hostname=bd6592c37bb2,40087,1732486423713, seqNum=5 because the exception is null or not the one we care about 2024-11-24T22:16:45,023 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732486585339.8ca644d2cada6dc44b363b7383dcd4fe., hostname=bd6592c37bb2,40087,1732486423713, seqNum=5 , the old value is region=TestAcidGuarantees,,1732486585339.8ca644d2cada6dc44b363b7383dcd4fe., hostname=bd6592c37bb2,40087,1732486423713, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8ca644d2cada6dc44b363b7383dcd4fe, server=bd6592c37bb2,40087,1732486423713 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T22:16:45,023 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732486585339.8ca644d2cada6dc44b363b7383dcd4fe., hostname=bd6592c37bb2,40087,1732486423713, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8ca644d2cada6dc44b363b7383dcd4fe, server=bd6592c37bb2,40087,1732486423713 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T22:16:45,023 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40087 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8ca644d2cada6dc44b363b7383dcd4fe, server=bd6592c37bb2,40087,1732486423713 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-24T22:16:45,023 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732486585339.8ca644d2cada6dc44b363b7383dcd4fe., hostname=bd6592c37bb2,40087,1732486423713, seqNum=5 because the exception is null or not the one we care about 2024-11-24T22:16:45,023 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40087 {}] ipc.CallRunner(138): callId: 191 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49002 deadline: 1732486665021, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8ca644d2cada6dc44b363b7383dcd4fe, server=bd6592c37bb2,40087,1732486423713 2024-11-24T22:16:45,024 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732486585339.8ca644d2cada6dc44b363b7383dcd4fe., hostname=bd6592c37bb2,40087,1732486423713, seqNum=5 , the old value is region=TestAcidGuarantees,,1732486585339.8ca644d2cada6dc44b363b7383dcd4fe., hostname=bd6592c37bb2,40087,1732486423713, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8ca644d2cada6dc44b363b7383dcd4fe, server=bd6592c37bb2,40087,1732486423713 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T22:16:45,024 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732486585339.8ca644d2cada6dc44b363b7383dcd4fe., hostname=bd6592c37bb2,40087,1732486423713, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8ca644d2cada6dc44b363b7383dcd4fe, server=bd6592c37bb2,40087,1732486423713 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T22:16:45,024 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732486585339.8ca644d2cada6dc44b363b7383dcd4fe., hostname=bd6592c37bb2,40087,1732486423713, seqNum=5 because the exception is null or not the one we care about 2024-11-24T22:16:45,132 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bd6592c37bb2:0-0 {event_type=RS_FLUSH_REGIONS, pid=163}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T22:16:45,135 INFO [RS_FLUSH_OPERATIONS-regionserver/bd6592c37bb2:0-0 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411248ec87d903bc94f8089f0ac1a67cd13d1_8ca644d2cada6dc44b363b7383dcd4fe to hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411248ec87d903bc94f8089f0ac1a67cd13d1_8ca644d2cada6dc44b363b7383dcd4fe 2024-11-24T22:16:45,136 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bd6592c37bb2:0-0 {event_type=RS_FLUSH_REGIONS, pid=163}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/8ca644d2cada6dc44b363b7383dcd4fe/.tmp/A/c18f1dc8317f4922af06f6e37bc834ac, store: [table=TestAcidGuarantees family=A region=8ca644d2cada6dc44b363b7383dcd4fe] 2024-11-24T22:16:45,137 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bd6592c37bb2:0-0 {event_type=RS_FLUSH_REGIONS, pid=163}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/8ca644d2cada6dc44b363b7383dcd4fe/.tmp/A/c18f1dc8317f4922af06f6e37bc834ac is 175, key is test_row_0/A:col10/1732486603762/Put/seqid=0 2024-11-24T22:16:45,146 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45929 is added to blk_1073742455_1631 (size=31255) 2024-11-24T22:16:45,147 INFO [RS_FLUSH_OPERATIONS-regionserver/bd6592c37bb2:0-0 {event_type=RS_FLUSH_REGIONS, pid=163}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=733, memsize=44.7 K, hasBloomFilter=true, into tmp file hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/8ca644d2cada6dc44b363b7383dcd4fe/.tmp/A/c18f1dc8317f4922af06f6e37bc834ac 2024-11-24T22:16:45,155 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bd6592c37bb2:0-0 {event_type=RS_FLUSH_REGIONS, pid=163}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/8ca644d2cada6dc44b363b7383dcd4fe/.tmp/B/e868df9854d3473fa99b4edb9eb7e4b9 is 50, key is test_row_0/B:col10/1732486603762/Put/seqid=0 2024-11-24T22:16:45,166 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45929 is added to blk_1073742456_1632 (size=12301) 2024-11-24T22:16:45,170 INFO [RS_FLUSH_OPERATIONS-regionserver/bd6592c37bb2:0-0 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=733 (bloomFilter=true), to=hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/8ca644d2cada6dc44b363b7383dcd4fe/.tmp/B/e868df9854d3473fa99b4edb9eb7e4b9 2024-11-24T22:16:45,185 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bd6592c37bb2:0-0 {event_type=RS_FLUSH_REGIONS, pid=163}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/8ca644d2cada6dc44b363b7383dcd4fe/.tmp/C/05a2a98b5f2043a6a93bfa785195587f is 50, key is test_row_0/C:col10/1732486603762/Put/seqid=0 2024-11-24T22:16:45,193 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45929 is added to blk_1073742457_1633 (size=12301) 2024-11-24T22:16:45,235 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40087 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8ca644d2cada6dc44b363b7383dcd4fe, server=bd6592c37bb2,40087,1732486423713 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-24T22:16:45,235 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40087 {}] ipc.CallRunner(138): callId: 201 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48986 deadline: 1732486665231, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8ca644d2cada6dc44b363b7383dcd4fe, server=bd6592c37bb2,40087,1732486423713 2024-11-24T22:16:45,235 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40087 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8ca644d2cada6dc44b363b7383dcd4fe, server=bd6592c37bb2,40087,1732486423713 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-24T22:16:45,235 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40087 {}] ipc.CallRunner(138): callId: 118 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49026 deadline: 1732486665231, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8ca644d2cada6dc44b363b7383dcd4fe, server=bd6592c37bb2,40087,1732486423713 2024-11-24T22:16:45,235 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732486585339.8ca644d2cada6dc44b363b7383dcd4fe., hostname=bd6592c37bb2,40087,1732486423713, seqNum=5 , the old value is region=TestAcidGuarantees,,1732486585339.8ca644d2cada6dc44b363b7383dcd4fe., hostname=bd6592c37bb2,40087,1732486423713, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8ca644d2cada6dc44b363b7383dcd4fe, server=bd6592c37bb2,40087,1732486423713 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T22:16:45,235 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732486585339.8ca644d2cada6dc44b363b7383dcd4fe., hostname=bd6592c37bb2,40087,1732486423713, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8ca644d2cada6dc44b363b7383dcd4fe, server=bd6592c37bb2,40087,1732486423713 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T22:16:45,235 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732486585339.8ca644d2cada6dc44b363b7383dcd4fe., hostname=bd6592c37bb2,40087,1732486423713, seqNum=5 because the exception is null or not the one we care about 2024-11-24T22:16:45,235 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40087 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8ca644d2cada6dc44b363b7383dcd4fe, server=bd6592c37bb2,40087,1732486423713 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-24T22:16:45,235 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732486585339.8ca644d2cada6dc44b363b7383dcd4fe., hostname=bd6592c37bb2,40087,1732486423713, seqNum=5 , the old value is region=TestAcidGuarantees,,1732486585339.8ca644d2cada6dc44b363b7383dcd4fe., hostname=bd6592c37bb2,40087,1732486423713, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8ca644d2cada6dc44b363b7383dcd4fe, server=bd6592c37bb2,40087,1732486423713 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T22:16:45,236 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732486585339.8ca644d2cada6dc44b363b7383dcd4fe., hostname=bd6592c37bb2,40087,1732486423713, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8ca644d2cada6dc44b363b7383dcd4fe, server=bd6592c37bb2,40087,1732486423713 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T22:16:45,236 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40087 {}] ipc.CallRunner(138): callId: 192 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49002 deadline: 1732486665231, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8ca644d2cada6dc44b363b7383dcd4fe, server=bd6592c37bb2,40087,1732486423713 2024-11-24T22:16:45,236 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732486585339.8ca644d2cada6dc44b363b7383dcd4fe., hostname=bd6592c37bb2,40087,1732486423713, seqNum=5 because the exception is null or not the one we care about 2024-11-24T22:16:45,236 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732486585339.8ca644d2cada6dc44b363b7383dcd4fe., hostname=bd6592c37bb2,40087,1732486423713, seqNum=5 , the old value is region=TestAcidGuarantees,,1732486585339.8ca644d2cada6dc44b363b7383dcd4fe., hostname=bd6592c37bb2,40087,1732486423713, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8ca644d2cada6dc44b363b7383dcd4fe, server=bd6592c37bb2,40087,1732486423713 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T22:16:45,236 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732486585339.8ca644d2cada6dc44b363b7383dcd4fe., hostname=bd6592c37bb2,40087,1732486423713, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8ca644d2cada6dc44b363b7383dcd4fe, server=bd6592c37bb2,40087,1732486423713 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T22:16:45,236 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732486585339.8ca644d2cada6dc44b363b7383dcd4fe., hostname=bd6592c37bb2,40087,1732486423713, seqNum=5 because the exception is null or not the one we care about 2024-11-24T22:16:45,363 DEBUG [regionserver/bd6592c37bb2:0.Chore.1 {}] throttle.PressureAwareCompactionThroughputController(103): CompactionPressure is 0.0, tune throughput to 50.00 MB/second 2024-11-24T22:16:45,544 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40087 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8ca644d2cada6dc44b363b7383dcd4fe, server=bd6592c37bb2,40087,1732486423713 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-24T22:16:45,544 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40087 {}] ipc.CallRunner(138): callId: 202 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48986 deadline: 1732486665541, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8ca644d2cada6dc44b363b7383dcd4fe, server=bd6592c37bb2,40087,1732486423713 2024-11-24T22:16:45,544 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40087 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8ca644d2cada6dc44b363b7383dcd4fe, server=bd6592c37bb2,40087,1732486423713 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-24T22:16:45,544 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40087 {}] ipc.CallRunner(138): callId: 119 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49026 deadline: 1732486665541, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8ca644d2cada6dc44b363b7383dcd4fe, server=bd6592c37bb2,40087,1732486423713 2024-11-24T22:16:45,544 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732486585339.8ca644d2cada6dc44b363b7383dcd4fe., hostname=bd6592c37bb2,40087,1732486423713, seqNum=5 , the old value is region=TestAcidGuarantees,,1732486585339.8ca644d2cada6dc44b363b7383dcd4fe., hostname=bd6592c37bb2,40087,1732486423713, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8ca644d2cada6dc44b363b7383dcd4fe, server=bd6592c37bb2,40087,1732486423713 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T22:16:45,544 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732486585339.8ca644d2cada6dc44b363b7383dcd4fe., hostname=bd6592c37bb2,40087,1732486423713, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8ca644d2cada6dc44b363b7383dcd4fe, server=bd6592c37bb2,40087,1732486423713 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T22:16:45,544 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732486585339.8ca644d2cada6dc44b363b7383dcd4fe., hostname=bd6592c37bb2,40087,1732486423713, seqNum=5 because the exception is null or not the one we care about 2024-11-24T22:16:45,544 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40087 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8ca644d2cada6dc44b363b7383dcd4fe, server=bd6592c37bb2,40087,1732486423713 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-24T22:16:45,545 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732486585339.8ca644d2cada6dc44b363b7383dcd4fe., hostname=bd6592c37bb2,40087,1732486423713, seqNum=5 , the old value is region=TestAcidGuarantees,,1732486585339.8ca644d2cada6dc44b363b7383dcd4fe., hostname=bd6592c37bb2,40087,1732486423713, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8ca644d2cada6dc44b363b7383dcd4fe, server=bd6592c37bb2,40087,1732486423713 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T22:16:45,545 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40087 {}] ipc.CallRunner(138): callId: 193 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49002 deadline: 1732486665541, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8ca644d2cada6dc44b363b7383dcd4fe, server=bd6592c37bb2,40087,1732486423713 2024-11-24T22:16:45,545 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732486585339.8ca644d2cada6dc44b363b7383dcd4fe., hostname=bd6592c37bb2,40087,1732486423713, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8ca644d2cada6dc44b363b7383dcd4fe, server=bd6592c37bb2,40087,1732486423713 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T22:16:45,545 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732486585339.8ca644d2cada6dc44b363b7383dcd4fe., hostname=bd6592c37bb2,40087,1732486423713, seqNum=5 because the exception is null or not the one we care about 2024-11-24T22:16:45,545 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732486585339.8ca644d2cada6dc44b363b7383dcd4fe., hostname=bd6592c37bb2,40087,1732486423713, seqNum=5 , the old value is region=TestAcidGuarantees,,1732486585339.8ca644d2cada6dc44b363b7383dcd4fe., hostname=bd6592c37bb2,40087,1732486423713, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8ca644d2cada6dc44b363b7383dcd4fe, server=bd6592c37bb2,40087,1732486423713 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T22:16:45,545 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732486585339.8ca644d2cada6dc44b363b7383dcd4fe., hostname=bd6592c37bb2,40087,1732486423713, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8ca644d2cada6dc44b363b7383dcd4fe, server=bd6592c37bb2,40087,1732486423713 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T22:16:45,545 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732486585339.8ca644d2cada6dc44b363b7383dcd4fe., hostname=bd6592c37bb2,40087,1732486423713, seqNum=5 because the exception is null or not the one we care about 2024-11-24T22:16:45,594 INFO [RS_FLUSH_OPERATIONS-regionserver/bd6592c37bb2:0-0 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=733 (bloomFilter=true), to=hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/8ca644d2cada6dc44b363b7383dcd4fe/.tmp/C/05a2a98b5f2043a6a93bfa785195587f 2024-11-24T22:16:45,600 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bd6592c37bb2:0-0 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/8ca644d2cada6dc44b363b7383dcd4fe/.tmp/A/c18f1dc8317f4922af06f6e37bc834ac as hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/8ca644d2cada6dc44b363b7383dcd4fe/A/c18f1dc8317f4922af06f6e37bc834ac 2024-11-24T22:16:45,604 INFO [RS_FLUSH_OPERATIONS-regionserver/bd6592c37bb2:0-0 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/8ca644d2cada6dc44b363b7383dcd4fe/A/c18f1dc8317f4922af06f6e37bc834ac, entries=150, sequenceid=733, filesize=30.5 K 2024-11-24T22:16:45,606 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bd6592c37bb2:0-0 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/8ca644d2cada6dc44b363b7383dcd4fe/.tmp/B/e868df9854d3473fa99b4edb9eb7e4b9 as hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/8ca644d2cada6dc44b363b7383dcd4fe/B/e868df9854d3473fa99b4edb9eb7e4b9 2024-11-24T22:16:45,610 INFO [RS_FLUSH_OPERATIONS-regionserver/bd6592c37bb2:0-0 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/8ca644d2cada6dc44b363b7383dcd4fe/B/e868df9854d3473fa99b4edb9eb7e4b9, entries=150, sequenceid=733, filesize=12.0 K 2024-11-24T22:16:45,611 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bd6592c37bb2:0-0 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/8ca644d2cada6dc44b363b7383dcd4fe/.tmp/C/05a2a98b5f2043a6a93bfa785195587f as hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/8ca644d2cada6dc44b363b7383dcd4fe/C/05a2a98b5f2043a6a93bfa785195587f 2024-11-24T22:16:45,615 INFO [RS_FLUSH_OPERATIONS-regionserver/bd6592c37bb2:0-0 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/8ca644d2cada6dc44b363b7383dcd4fe/C/05a2a98b5f2043a6a93bfa785195587f, entries=150, sequenceid=733, filesize=12.0 K 2024-11-24T22:16:45,616 INFO [RS_FLUSH_OPERATIONS-regionserver/bd6592c37bb2:0-0 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.HRegion(3140): Finished flush of dataSize ~134.18 KB/137400, heapSize ~375.56 KB/384576, currentSize=46.96 KB/48090 for 8ca644d2cada6dc44b363b7383dcd4fe in 950ms, sequenceid=733, compaction requested=true 2024-11-24T22:16:45,616 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bd6592c37bb2:0-0 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.HRegion(2603): Flush status journal for 8ca644d2cada6dc44b363b7383dcd4fe: 2024-11-24T22:16:45,616 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bd6592c37bb2:0-0 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732486585339.8ca644d2cada6dc44b363b7383dcd4fe. 2024-11-24T22:16:45,616 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bd6592c37bb2:0-0 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=163 2024-11-24T22:16:45,616 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37701 {}] master.HMaster(4169): Remote procedure done, pid=163 2024-11-24T22:16:45,620 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=163, resume processing ppid=162 2024-11-24T22:16:45,620 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=163, ppid=162, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 3.4030 sec 2024-11-24T22:16:45,621 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=162, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=162, table=TestAcidGuarantees in 3.4080 sec 2024-11-24T22:16:45,928 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40087 {}] regionserver.HRegion(8855): Flush requested on 8ca644d2cada6dc44b363b7383dcd4fe 2024-11-24T22:16:45,928 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 8ca644d2cada6dc44b363b7383dcd4fe 3/3 column families, dataSize=53.67 KB heapSize=160.13 KB 2024-11-24T22:16:45,928 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 8ca644d2cada6dc44b363b7383dcd4fe, store=A 2024-11-24T22:16:45,928 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-24T22:16:45,928 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 8ca644d2cada6dc44b363b7383dcd4fe, store=B 2024-11-24T22:16:45,928 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-24T22:16:45,928 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 8ca644d2cada6dc44b363b7383dcd4fe, store=C 2024-11-24T22:16:45,928 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-24T22:16:45,942 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241124587d91e3431c4d7e948bb918733ef8d3_8ca644d2cada6dc44b363b7383dcd4fe is 50, key is test_row_0/A:col10/1732486604914/Put/seqid=0 2024-11-24T22:16:45,963 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45929 is added to blk_1073742458_1634 (size=14994) 2024-11-24T22:16:45,963 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T22:16:45,966 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241124587d91e3431c4d7e948bb918733ef8d3_8ca644d2cada6dc44b363b7383dcd4fe to hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241124587d91e3431c4d7e948bb918733ef8d3_8ca644d2cada6dc44b363b7383dcd4fe 2024-11-24T22:16:45,968 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/8ca644d2cada6dc44b363b7383dcd4fe/.tmp/A/18acc92875294311b85e9cf8ff590296, store: [table=TestAcidGuarantees family=A region=8ca644d2cada6dc44b363b7383dcd4fe] 2024-11-24T22:16:45,969 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/8ca644d2cada6dc44b363b7383dcd4fe/.tmp/A/18acc92875294311b85e9cf8ff590296 is 175, key is test_row_0/A:col10/1732486604914/Put/seqid=0 2024-11-24T22:16:45,977 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45929 is added to blk_1073742459_1635 (size=39949) 2024-11-24T22:16:45,978 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=744, memsize=17.9 K, hasBloomFilter=true, into tmp file hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/8ca644d2cada6dc44b363b7383dcd4fe/.tmp/A/18acc92875294311b85e9cf8ff590296 2024-11-24T22:16:45,986 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/8ca644d2cada6dc44b363b7383dcd4fe/.tmp/B/1eaf0718bad24a90ab90b86f3a35f33f is 50, key is test_row_0/B:col10/1732486604914/Put/seqid=0 2024-11-24T22:16:45,993 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45929 is added to blk_1073742460_1636 (size=12301) 2024-11-24T22:16:45,994 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=744 (bloomFilter=true), to=hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/8ca644d2cada6dc44b363b7383dcd4fe/.tmp/B/1eaf0718bad24a90ab90b86f3a35f33f 2024-11-24T22:16:45,998 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/bd6592c37bb2:0-5 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=209, cellSize=9565, totalHeapSize=21976, min timestamp=1732486605947, max timestamp=1732486605995Num uniques 859; 2024-11-24T22:16:45,998 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/bd6592c37bb2:0-5 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=9565, suffix heap size=117064,new segment heap size=21976 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=209 2024-11-24T22:16:45,998 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/bd6592c37bb2:0-7 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=209, cellSize=9565, totalHeapSize=21976, min timestamp=1732486605947, max timestamp=1732486605995Num uniques 859; 2024-11-24T22:16:45,998 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/bd6592c37bb2:0-7 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=9565, suffix heap size=117064,new segment heap size=21976 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=209 2024-11-24T22:16:46,001 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/bd6592c37bb2:0-8 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1732486605947, max timestamp=1732486605995Num uniques 859; 2024-11-24T22:16:46,001 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/bd6592c37bb2:0-8 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-11-24T22:16:46,007 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/8ca644d2cada6dc44b363b7383dcd4fe/.tmp/C/a88236e6fd224409ad07b616601f72ff is 50, key is test_row_0/C:col10/1732486604914/Put/seqid=0 2024-11-24T22:16:46,025 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45929 is added to blk_1073742461_1637 (size=12301) 2024-11-24T22:16:46,027 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=744 (bloomFilter=true), to=hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/8ca644d2cada6dc44b363b7383dcd4fe/.tmp/C/a88236e6fd224409ad07b616601f72ff 2024-11-24T22:16:46,032 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/8ca644d2cada6dc44b363b7383dcd4fe/.tmp/A/18acc92875294311b85e9cf8ff590296 as hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/8ca644d2cada6dc44b363b7383dcd4fe/A/18acc92875294311b85e9cf8ff590296 2024-11-24T22:16:46,040 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/8ca644d2cada6dc44b363b7383dcd4fe/A/18acc92875294311b85e9cf8ff590296, entries=200, sequenceid=744, filesize=39.0 K 2024-11-24T22:16:46,040 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/8ca644d2cada6dc44b363b7383dcd4fe/.tmp/B/1eaf0718bad24a90ab90b86f3a35f33f as hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/8ca644d2cada6dc44b363b7383dcd4fe/B/1eaf0718bad24a90ab90b86f3a35f33f 2024-11-24T22:16:46,046 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/8ca644d2cada6dc44b363b7383dcd4fe/B/1eaf0718bad24a90ab90b86f3a35f33f, entries=150, sequenceid=744, filesize=12.0 K 2024-11-24T22:16:46,047 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/8ca644d2cada6dc44b363b7383dcd4fe/.tmp/C/a88236e6fd224409ad07b616601f72ff as hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/8ca644d2cada6dc44b363b7383dcd4fe/C/a88236e6fd224409ad07b616601f72ff 2024-11-24T22:16:46,050 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/8ca644d2cada6dc44b363b7383dcd4fe/C/a88236e6fd224409ad07b616601f72ff, entries=150, sequenceid=744, filesize=12.0 K 2024-11-24T22:16:46,052 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~53.67 KB/54960, heapSize ~160.08 KB/163920, currentSize=105.11 KB/107630 for 8ca644d2cada6dc44b363b7383dcd4fe in 124ms, sequenceid=744, compaction requested=true 2024-11-24T22:16:46,052 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 8ca644d2cada6dc44b363b7383dcd4fe: 2024-11-24T22:16:46,052 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 8ca644d2cada6dc44b363b7383dcd4fe:A, priority=-2147483648, current under compaction store size is 1 2024-11-24T22:16:46,052 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-24T22:16:46,052 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 8ca644d2cada6dc44b363b7383dcd4fe:B, priority=-2147483648, current under compaction store size is 2 2024-11-24T22:16:46,052 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-24T22:16:46,052 DEBUG [RS:0;bd6592c37bb2:40087-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-24T22:16:46,052 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 8ca644d2cada6dc44b363b7383dcd4fe:C, priority=-2147483648, current under compaction store size is 3 2024-11-24T22:16:46,052 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=2), splitQueue=0 2024-11-24T22:16:46,052 DEBUG [RS:0;bd6592c37bb2:40087-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-24T22:16:46,053 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40087 {}] regionserver.HRegion(8855): Flush requested on 8ca644d2cada6dc44b363b7383dcd4fe 2024-11-24T22:16:46,053 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 8ca644d2cada6dc44b363b7383dcd4fe 3/3 column families, dataSize=111.82 KB heapSize=315.45 KB 2024-11-24T22:16:46,053 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 8ca644d2cada6dc44b363b7383dcd4fe, store=A 2024-11-24T22:16:46,053 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-24T22:16:46,053 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 8ca644d2cada6dc44b363b7383dcd4fe, store=B 2024-11-24T22:16:46,053 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-24T22:16:46,053 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 8ca644d2cada6dc44b363b7383dcd4fe, store=C 2024-11-24T22:16:46,053 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-24T22:16:46,055 DEBUG [RS:0;bd6592c37bb2:40087-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 49566 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-24T22:16:46,055 DEBUG [RS:0;bd6592c37bb2:40087-longCompactions-0 {}] regionserver.HStore(1541): 8ca644d2cada6dc44b363b7383dcd4fe/B is initiating minor compaction (all files) 2024-11-24T22:16:46,055 INFO [RS:0;bd6592c37bb2:40087-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 8ca644d2cada6dc44b363b7383dcd4fe/B in TestAcidGuarantees,,1732486585339.8ca644d2cada6dc44b363b7383dcd4fe. 2024-11-24T22:16:46,055 INFO [RS:0;bd6592c37bb2:40087-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/8ca644d2cada6dc44b363b7383dcd4fe/B/dc23ac5e77d74dc0906774d8a726505d, hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/8ca644d2cada6dc44b363b7383dcd4fe/B/6b52569f4db642aebd605b9ea3871fa5, hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/8ca644d2cada6dc44b363b7383dcd4fe/B/e868df9854d3473fa99b4edb9eb7e4b9, hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/8ca644d2cada6dc44b363b7383dcd4fe/B/1eaf0718bad24a90ab90b86f3a35f33f] into tmpdir=hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/8ca644d2cada6dc44b363b7383dcd4fe/.tmp, totalSize=48.4 K 2024-11-24T22:16:46,055 DEBUG [RS:0;bd6592c37bb2:40087-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 134076 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-24T22:16:46,056 DEBUG [RS:0;bd6592c37bb2:40087-shortCompactions-0 {}] regionserver.HStore(1541): 8ca644d2cada6dc44b363b7383dcd4fe/A is initiating minor compaction (all files) 2024-11-24T22:16:46,056 INFO [RS:0;bd6592c37bb2:40087-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 8ca644d2cada6dc44b363b7383dcd4fe/A in TestAcidGuarantees,,1732486585339.8ca644d2cada6dc44b363b7383dcd4fe. 2024-11-24T22:16:46,056 INFO [RS:0;bd6592c37bb2:40087-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/8ca644d2cada6dc44b363b7383dcd4fe/A/7263a115caa04d7d9b7c84d0a6c9cce4, hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/8ca644d2cada6dc44b363b7383dcd4fe/A/99464ebe699642cfa110755eb79a0020, hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/8ca644d2cada6dc44b363b7383dcd4fe/A/c18f1dc8317f4922af06f6e37bc834ac, hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/8ca644d2cada6dc44b363b7383dcd4fe/A/18acc92875294311b85e9cf8ff590296] into tmpdir=hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/8ca644d2cada6dc44b363b7383dcd4fe/.tmp, totalSize=130.9 K 2024-11-24T22:16:46,056 INFO [RS:0;bd6592c37bb2:40087-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=12 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1732486585339.8ca644d2cada6dc44b363b7383dcd4fe. 2024-11-24T22:16:46,056 DEBUG [RS:0;bd6592c37bb2:40087-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1732486585339.8ca644d2cada6dc44b363b7383dcd4fe. files: [hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/8ca644d2cada6dc44b363b7383dcd4fe/A/7263a115caa04d7d9b7c84d0a6c9cce4, hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/8ca644d2cada6dc44b363b7383dcd4fe/A/99464ebe699642cfa110755eb79a0020, hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/8ca644d2cada6dc44b363b7383dcd4fe/A/c18f1dc8317f4922af06f6e37bc834ac, hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/8ca644d2cada6dc44b363b7383dcd4fe/A/18acc92875294311b85e9cf8ff590296] 2024-11-24T22:16:46,056 DEBUG [RS:0;bd6592c37bb2:40087-longCompactions-0 {}] compactions.Compactor(225): Compacting dc23ac5e77d74dc0906774d8a726505d, keycount=150, bloomtype=ROW, size=12.4 K, encoding=NONE, compression=NONE, seqNum=225, earliestPutTs=1732486601981 2024-11-24T22:16:46,056 DEBUG [RS:0;bd6592c37bb2:40087-shortCompactions-0 {}] compactions.Compactor(225): Compacting 7263a115caa04d7d9b7c84d0a6c9cce4, keycount=150, bloomtype=ROW, size=30.9 K, encoding=NONE, compression=NONE, seqNum=225, earliestPutTs=1732486601981 2024-11-24T22:16:46,056 DEBUG [RS:0;bd6592c37bb2:40087-longCompactions-0 {}] compactions.Compactor(225): Compacting 6b52569f4db642aebd605b9ea3871fa5, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=695, earliestPutTs=1732486603614 2024-11-24T22:16:46,056 DEBUG [RS:0;bd6592c37bb2:40087-shortCompactions-0 {}] compactions.Compactor(225): Compacting 99464ebe699642cfa110755eb79a0020, keycount=150, bloomtype=ROW, size=30.5 K, encoding=NONE, compression=NONE, seqNum=695, earliestPutTs=1732486603614 2024-11-24T22:16:46,057 DEBUG [RS:0;bd6592c37bb2:40087-longCompactions-0 {}] compactions.Compactor(225): Compacting e868df9854d3473fa99b4edb9eb7e4b9, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=733, earliestPutTs=1732486603755 2024-11-24T22:16:46,057 DEBUG [RS:0;bd6592c37bb2:40087-shortCompactions-0 {}] compactions.Compactor(225): Compacting c18f1dc8317f4922af06f6e37bc834ac, keycount=150, bloomtype=ROW, size=30.5 K, encoding=NONE, compression=NONE, seqNum=733, earliestPutTs=1732486603755 2024-11-24T22:16:46,057 DEBUG [RS:0;bd6592c37bb2:40087-longCompactions-0 {}] compactions.Compactor(225): Compacting 1eaf0718bad24a90ab90b86f3a35f33f, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=744, earliestPutTs=1732486604909 2024-11-24T22:16:46,057 DEBUG [RS:0;bd6592c37bb2:40087-shortCompactions-0 {}] compactions.Compactor(225): Compacting 18acc92875294311b85e9cf8ff590296, keycount=200, bloomtype=ROW, size=39.0 K, encoding=NONE, compression=NONE, seqNum=744, earliestPutTs=1732486604909 2024-11-24T22:16:46,074 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40087 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8ca644d2cada6dc44b363b7383dcd4fe, server=bd6592c37bb2,40087,1732486423713 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-24T22:16:46,074 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40087 {}] ipc.CallRunner(138): callId: 231 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49050 deadline: 1732486666065, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8ca644d2cada6dc44b363b7383dcd4fe, server=bd6592c37bb2,40087,1732486423713 2024-11-24T22:16:46,075 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732486585339.8ca644d2cada6dc44b363b7383dcd4fe., hostname=bd6592c37bb2,40087,1732486423713, seqNum=5 , the old value is region=TestAcidGuarantees,,1732486585339.8ca644d2cada6dc44b363b7383dcd4fe., hostname=bd6592c37bb2,40087,1732486423713, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8ca644d2cada6dc44b363b7383dcd4fe, server=bd6592c37bb2,40087,1732486423713 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T22:16:46,075 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732486585339.8ca644d2cada6dc44b363b7383dcd4fe., hostname=bd6592c37bb2,40087,1732486423713, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8ca644d2cada6dc44b363b7383dcd4fe, server=bd6592c37bb2,40087,1732486423713 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T22:16:46,075 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732486585339.8ca644d2cada6dc44b363b7383dcd4fe., hostname=bd6592c37bb2,40087,1732486423713, seqNum=5 because the exception is null or not the one we care about 2024-11-24T22:16:46,077 INFO [RS:0;bd6592c37bb2:40087-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=8ca644d2cada6dc44b363b7383dcd4fe] 2024-11-24T22:16:46,079 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40087 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8ca644d2cada6dc44b363b7383dcd4fe, server=bd6592c37bb2,40087,1732486423713 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-24T22:16:46,079 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40087 {}] ipc.CallRunner(138): callId: 122 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49026 deadline: 1732486666070, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8ca644d2cada6dc44b363b7383dcd4fe, server=bd6592c37bb2,40087,1732486423713 2024-11-24T22:16:46,080 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732486585339.8ca644d2cada6dc44b363b7383dcd4fe., hostname=bd6592c37bb2,40087,1732486423713, seqNum=5 , the old value is region=TestAcidGuarantees,,1732486585339.8ca644d2cada6dc44b363b7383dcd4fe., hostname=bd6592c37bb2,40087,1732486423713, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8ca644d2cada6dc44b363b7383dcd4fe, server=bd6592c37bb2,40087,1732486423713 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T22:16:46,080 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732486585339.8ca644d2cada6dc44b363b7383dcd4fe., hostname=bd6592c37bb2,40087,1732486423713, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8ca644d2cada6dc44b363b7383dcd4fe, server=bd6592c37bb2,40087,1732486423713 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T22:16:46,080 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40087 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8ca644d2cada6dc44b363b7383dcd4fe, server=bd6592c37bb2,40087,1732486423713 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-24T22:16:46,080 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732486585339.8ca644d2cada6dc44b363b7383dcd4fe., hostname=bd6592c37bb2,40087,1732486423713, seqNum=5 because the exception is null or not the one we care about 2024-11-24T22:16:46,080 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40087 {}] ipc.CallRunner(138): callId: 205 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48986 deadline: 1732486666071, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8ca644d2cada6dc44b363b7383dcd4fe, server=bd6592c37bb2,40087,1732486423713 2024-11-24T22:16:46,080 INFO [RS:0;bd6592c37bb2:40087-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 8ca644d2cada6dc44b363b7383dcd4fe#B#compaction#537 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-24T22:16:46,080 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732486585339.8ca644d2cada6dc44b363b7383dcd4fe., hostname=bd6592c37bb2,40087,1732486423713, seqNum=5 , the old value is region=TestAcidGuarantees,,1732486585339.8ca644d2cada6dc44b363b7383dcd4fe., hostname=bd6592c37bb2,40087,1732486423713, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8ca644d2cada6dc44b363b7383dcd4fe, server=bd6592c37bb2,40087,1732486423713 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T22:16:46,080 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732486585339.8ca644d2cada6dc44b363b7383dcd4fe., hostname=bd6592c37bb2,40087,1732486423713, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8ca644d2cada6dc44b363b7383dcd4fe, server=bd6592c37bb2,40087,1732486423713 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T22:16:46,080 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732486585339.8ca644d2cada6dc44b363b7383dcd4fe., hostname=bd6592c37bb2,40087,1732486423713, seqNum=5 because the exception is null or not the one we care about 2024-11-24T22:16:46,080 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241124c28a62c49349432aaa77d1513ea0f9c4_8ca644d2cada6dc44b363b7383dcd4fe is 50, key is test_row_0/A:col10/1732486606021/Put/seqid=0 2024-11-24T22:16:46,080 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40087 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8ca644d2cada6dc44b363b7383dcd4fe, server=bd6592c37bb2,40087,1732486423713 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-24T22:16:46,081 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40087 {}] ipc.CallRunner(138): callId: 196 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49002 deadline: 1732486666073, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8ca644d2cada6dc44b363b7383dcd4fe, server=bd6592c37bb2,40087,1732486423713 2024-11-24T22:16:46,081 DEBUG [RS:0;bd6592c37bb2:40087-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/8ca644d2cada6dc44b363b7383dcd4fe/.tmp/B/3b068b24fdb74c6190094f2727449194 is 50, key is test_row_0/B:col10/1732486604914/Put/seqid=0 2024-11-24T22:16:46,081 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732486585339.8ca644d2cada6dc44b363b7383dcd4fe., hostname=bd6592c37bb2,40087,1732486423713, seqNum=5 , the old value is region=TestAcidGuarantees,,1732486585339.8ca644d2cada6dc44b363b7383dcd4fe., hostname=bd6592c37bb2,40087,1732486423713, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8ca644d2cada6dc44b363b7383dcd4fe, server=bd6592c37bb2,40087,1732486423713 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T22:16:46,081 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732486585339.8ca644d2cada6dc44b363b7383dcd4fe., hostname=bd6592c37bb2,40087,1732486423713, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8ca644d2cada6dc44b363b7383dcd4fe, server=bd6592c37bb2,40087,1732486423713 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T22:16:46,081 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732486585339.8ca644d2cada6dc44b363b7383dcd4fe., hostname=bd6592c37bb2,40087,1732486423713, seqNum=5 because the exception is null or not the one we care about 2024-11-24T22:16:46,082 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40087 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8ca644d2cada6dc44b363b7383dcd4fe, server=bd6592c37bb2,40087,1732486423713 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-24T22:16:46,082 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40087 {}] ipc.CallRunner(138): callId: 147 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49060 deadline: 1732486666075, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8ca644d2cada6dc44b363b7383dcd4fe, server=bd6592c37bb2,40087,1732486423713 2024-11-24T22:16:46,084 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732486585339.8ca644d2cada6dc44b363b7383dcd4fe., hostname=bd6592c37bb2,40087,1732486423713, seqNum=5 , the old value is region=TestAcidGuarantees,,1732486585339.8ca644d2cada6dc44b363b7383dcd4fe., hostname=bd6592c37bb2,40087,1732486423713, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8ca644d2cada6dc44b363b7383dcd4fe, server=bd6592c37bb2,40087,1732486423713 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T22:16:46,084 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732486585339.8ca644d2cada6dc44b363b7383dcd4fe., hostname=bd6592c37bb2,40087,1732486423713, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8ca644d2cada6dc44b363b7383dcd4fe, server=bd6592c37bb2,40087,1732486423713 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T22:16:46,084 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732486585339.8ca644d2cada6dc44b363b7383dcd4fe., hostname=bd6592c37bb2,40087,1732486423713, seqNum=5 because the exception is null or not the one we care about 2024-11-24T22:16:46,084 DEBUG [RS:0;bd6592c37bb2:40087-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e20241124bd7db71b43164c798963b96ac03ca4e8_8ca644d2cada6dc44b363b7383dcd4fe store=[table=TestAcidGuarantees family=A region=8ca644d2cada6dc44b363b7383dcd4fe] 2024-11-24T22:16:46,087 DEBUG [RS:0;bd6592c37bb2:40087-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e20241124bd7db71b43164c798963b96ac03ca4e8_8ca644d2cada6dc44b363b7383dcd4fe, store=[table=TestAcidGuarantees family=A region=8ca644d2cada6dc44b363b7383dcd4fe] 2024-11-24T22:16:46,087 DEBUG [RS:0;bd6592c37bb2:40087-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241124bd7db71b43164c798963b96ac03ca4e8_8ca644d2cada6dc44b363b7383dcd4fe because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=8ca644d2cada6dc44b363b7383dcd4fe] 2024-11-24T22:16:46,131 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45929 is added to blk_1073742462_1638 (size=14994) 2024-11-24T22:16:46,144 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45929 is added to blk_1073742464_1640 (size=4469) 2024-11-24T22:16:46,145 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45929 is added to blk_1073742463_1639 (size=12949) 2024-11-24T22:16:46,187 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40087 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8ca644d2cada6dc44b363b7383dcd4fe, server=bd6592c37bb2,40087,1732486423713 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-24T22:16:46,187 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40087 {}] ipc.CallRunner(138): callId: 232 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49050 deadline: 1732486666181, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8ca644d2cada6dc44b363b7383dcd4fe, server=bd6592c37bb2,40087,1732486423713 2024-11-24T22:16:46,187 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732486585339.8ca644d2cada6dc44b363b7383dcd4fe., hostname=bd6592c37bb2,40087,1732486423713, seqNum=5 , the old value is region=TestAcidGuarantees,,1732486585339.8ca644d2cada6dc44b363b7383dcd4fe., hostname=bd6592c37bb2,40087,1732486423713, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8ca644d2cada6dc44b363b7383dcd4fe, server=bd6592c37bb2,40087,1732486423713 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T22:16:46,187 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732486585339.8ca644d2cada6dc44b363b7383dcd4fe., hostname=bd6592c37bb2,40087,1732486423713, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8ca644d2cada6dc44b363b7383dcd4fe, server=bd6592c37bb2,40087,1732486423713 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T22:16:46,187 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732486585339.8ca644d2cada6dc44b363b7383dcd4fe., hostname=bd6592c37bb2,40087,1732486423713, seqNum=5 because the exception is null or not the one we care about 2024-11-24T22:16:46,196 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40087 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8ca644d2cada6dc44b363b7383dcd4fe, server=bd6592c37bb2,40087,1732486423713 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-24T22:16:46,197 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40087 {}] ipc.CallRunner(138): callId: 123 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49026 deadline: 1732486666191, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8ca644d2cada6dc44b363b7383dcd4fe, server=bd6592c37bb2,40087,1732486423713 2024-11-24T22:16:46,197 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40087 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8ca644d2cada6dc44b363b7383dcd4fe, server=bd6592c37bb2,40087,1732486423713 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-24T22:16:46,197 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732486585339.8ca644d2cada6dc44b363b7383dcd4fe., hostname=bd6592c37bb2,40087,1732486423713, seqNum=5 , the old value is region=TestAcidGuarantees,,1732486585339.8ca644d2cada6dc44b363b7383dcd4fe., hostname=bd6592c37bb2,40087,1732486423713, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8ca644d2cada6dc44b363b7383dcd4fe, server=bd6592c37bb2,40087,1732486423713 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T22:16:46,197 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732486585339.8ca644d2cada6dc44b363b7383dcd4fe., hostname=bd6592c37bb2,40087,1732486423713, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8ca644d2cada6dc44b363b7383dcd4fe, server=bd6592c37bb2,40087,1732486423713 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T22:16:46,197 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732486585339.8ca644d2cada6dc44b363b7383dcd4fe., hostname=bd6592c37bb2,40087,1732486423713, seqNum=5 because the exception is null or not the one we care about 2024-11-24T22:16:46,197 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40087 {}] ipc.CallRunner(138): callId: 206 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48986 deadline: 1732486666191, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8ca644d2cada6dc44b363b7383dcd4fe, server=bd6592c37bb2,40087,1732486423713 2024-11-24T22:16:46,197 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732486585339.8ca644d2cada6dc44b363b7383dcd4fe., hostname=bd6592c37bb2,40087,1732486423713, seqNum=5 , the old value is region=TestAcidGuarantees,,1732486585339.8ca644d2cada6dc44b363b7383dcd4fe., hostname=bd6592c37bb2,40087,1732486423713, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8ca644d2cada6dc44b363b7383dcd4fe, server=bd6592c37bb2,40087,1732486423713 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T22:16:46,197 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732486585339.8ca644d2cada6dc44b363b7383dcd4fe., hostname=bd6592c37bb2,40087,1732486423713, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8ca644d2cada6dc44b363b7383dcd4fe, server=bd6592c37bb2,40087,1732486423713 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T22:16:46,197 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40087 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8ca644d2cada6dc44b363b7383dcd4fe, server=bd6592c37bb2,40087,1732486423713 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-24T22:16:46,197 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732486585339.8ca644d2cada6dc44b363b7383dcd4fe., hostname=bd6592c37bb2,40087,1732486423713, seqNum=5 because the exception is null or not the one we care about 2024-11-24T22:16:46,197 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40087 {}] ipc.CallRunner(138): callId: 197 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49002 deadline: 1732486666191, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8ca644d2cada6dc44b363b7383dcd4fe, server=bd6592c37bb2,40087,1732486423713 2024-11-24T22:16:46,198 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732486585339.8ca644d2cada6dc44b363b7383dcd4fe., hostname=bd6592c37bb2,40087,1732486423713, seqNum=5 , the old value is region=TestAcidGuarantees,,1732486585339.8ca644d2cada6dc44b363b7383dcd4fe., hostname=bd6592c37bb2,40087,1732486423713, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8ca644d2cada6dc44b363b7383dcd4fe, server=bd6592c37bb2,40087,1732486423713 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T22:16:46,198 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732486585339.8ca644d2cada6dc44b363b7383dcd4fe., hostname=bd6592c37bb2,40087,1732486423713, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8ca644d2cada6dc44b363b7383dcd4fe, server=bd6592c37bb2,40087,1732486423713 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T22:16:46,198 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732486585339.8ca644d2cada6dc44b363b7383dcd4fe., hostname=bd6592c37bb2,40087,1732486423713, seqNum=5 because the exception is null or not the one we care about 2024-11-24T22:16:46,198 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40087 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8ca644d2cada6dc44b363b7383dcd4fe, server=bd6592c37bb2,40087,1732486423713 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-24T22:16:46,198 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40087 {}] ipc.CallRunner(138): callId: 148 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49060 deadline: 1732486666191, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8ca644d2cada6dc44b363b7383dcd4fe, server=bd6592c37bb2,40087,1732486423713 2024-11-24T22:16:46,198 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732486585339.8ca644d2cada6dc44b363b7383dcd4fe., hostname=bd6592c37bb2,40087,1732486423713, seqNum=5 , the old value is region=TestAcidGuarantees,,1732486585339.8ca644d2cada6dc44b363b7383dcd4fe., hostname=bd6592c37bb2,40087,1732486423713, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8ca644d2cada6dc44b363b7383dcd4fe, server=bd6592c37bb2,40087,1732486423713 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T22:16:46,198 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732486585339.8ca644d2cada6dc44b363b7383dcd4fe., hostname=bd6592c37bb2,40087,1732486423713, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8ca644d2cada6dc44b363b7383dcd4fe, server=bd6592c37bb2,40087,1732486423713 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T22:16:46,198 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732486585339.8ca644d2cada6dc44b363b7383dcd4fe., hostname=bd6592c37bb2,40087,1732486423713, seqNum=5 because the exception is null or not the one we care about 2024-11-24T22:16:46,381 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37701 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=162 2024-11-24T22:16:46,381 INFO [RPCClient-NioEventLoopGroup-4-12 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-24T22:16:46,382 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37701 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-24T22:16:46,383 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37701 {}] procedure2.ProcedureExecutor(1139): Stored pid=164, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=164, table=TestAcidGuarantees 2024-11-24T22:16:46,383 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37701 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=164 2024-11-24T22:16:46,385 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=164, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=164, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-24T22:16:46,390 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=164, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=164, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-24T22:16:46,390 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=165, ppid=164, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-24T22:16:46,397 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40087 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8ca644d2cada6dc44b363b7383dcd4fe, server=bd6592c37bb2,40087,1732486423713 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-24T22:16:46,397 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40087 {}] ipc.CallRunner(138): callId: 233 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49050 deadline: 1732486666391, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8ca644d2cada6dc44b363b7383dcd4fe, server=bd6592c37bb2,40087,1732486423713 2024-11-24T22:16:46,397 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732486585339.8ca644d2cada6dc44b363b7383dcd4fe., hostname=bd6592c37bb2,40087,1732486423713, seqNum=5 , the old value is region=TestAcidGuarantees,,1732486585339.8ca644d2cada6dc44b363b7383dcd4fe., hostname=bd6592c37bb2,40087,1732486423713, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8ca644d2cada6dc44b363b7383dcd4fe, server=bd6592c37bb2,40087,1732486423713 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T22:16:46,397 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732486585339.8ca644d2cada6dc44b363b7383dcd4fe., hostname=bd6592c37bb2,40087,1732486423713, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8ca644d2cada6dc44b363b7383dcd4fe, server=bd6592c37bb2,40087,1732486423713 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T22:16:46,397 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732486585339.8ca644d2cada6dc44b363b7383dcd4fe., hostname=bd6592c37bb2,40087,1732486423713, seqNum=5 because the exception is null or not the one we care about 2024-11-24T22:16:46,407 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40087 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8ca644d2cada6dc44b363b7383dcd4fe, server=bd6592c37bb2,40087,1732486423713 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-24T22:16:46,407 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40087 {}] ipc.CallRunner(138): callId: 124 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49026 deadline: 1732486666401, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8ca644d2cada6dc44b363b7383dcd4fe, server=bd6592c37bb2,40087,1732486423713 2024-11-24T22:16:46,407 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40087 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8ca644d2cada6dc44b363b7383dcd4fe, server=bd6592c37bb2,40087,1732486423713 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-24T22:16:46,407 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732486585339.8ca644d2cada6dc44b363b7383dcd4fe., hostname=bd6592c37bb2,40087,1732486423713, seqNum=5 , the old value is region=TestAcidGuarantees,,1732486585339.8ca644d2cada6dc44b363b7383dcd4fe., hostname=bd6592c37bb2,40087,1732486423713, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8ca644d2cada6dc44b363b7383dcd4fe, server=bd6592c37bb2,40087,1732486423713 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T22:16:46,407 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732486585339.8ca644d2cada6dc44b363b7383dcd4fe., hostname=bd6592c37bb2,40087,1732486423713, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8ca644d2cada6dc44b363b7383dcd4fe, server=bd6592c37bb2,40087,1732486423713 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T22:16:46,407 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40087 {}] ipc.CallRunner(138): callId: 198 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49002 deadline: 1732486666402, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8ca644d2cada6dc44b363b7383dcd4fe, server=bd6592c37bb2,40087,1732486423713 2024-11-24T22:16:46,407 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732486585339.8ca644d2cada6dc44b363b7383dcd4fe., hostname=bd6592c37bb2,40087,1732486423713, seqNum=5 because the exception is null or not the one we care about 2024-11-24T22:16:46,407 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40087 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8ca644d2cada6dc44b363b7383dcd4fe, server=bd6592c37bb2,40087,1732486423713 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-24T22:16:46,407 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40087 {}] ipc.CallRunner(138): callId: 207 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48986 deadline: 1732486666402, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8ca644d2cada6dc44b363b7383dcd4fe, server=bd6592c37bb2,40087,1732486423713 2024-11-24T22:16:46,408 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732486585339.8ca644d2cada6dc44b363b7383dcd4fe., hostname=bd6592c37bb2,40087,1732486423713, seqNum=5 , the old value is region=TestAcidGuarantees,,1732486585339.8ca644d2cada6dc44b363b7383dcd4fe., hostname=bd6592c37bb2,40087,1732486423713, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8ca644d2cada6dc44b363b7383dcd4fe, server=bd6592c37bb2,40087,1732486423713 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T22:16:46,408 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732486585339.8ca644d2cada6dc44b363b7383dcd4fe., hostname=bd6592c37bb2,40087,1732486423713, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8ca644d2cada6dc44b363b7383dcd4fe, server=bd6592c37bb2,40087,1732486423713 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T22:16:46,408 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732486585339.8ca644d2cada6dc44b363b7383dcd4fe., hostname=bd6592c37bb2,40087,1732486423713, seqNum=5 because the exception is null or not the one we care about 2024-11-24T22:16:46,408 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40087 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8ca644d2cada6dc44b363b7383dcd4fe, server=bd6592c37bb2,40087,1732486423713 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-24T22:16:46,408 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732486585339.8ca644d2cada6dc44b363b7383dcd4fe., hostname=bd6592c37bb2,40087,1732486423713, seqNum=5 , the old value is region=TestAcidGuarantees,,1732486585339.8ca644d2cada6dc44b363b7383dcd4fe., hostname=bd6592c37bb2,40087,1732486423713, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8ca644d2cada6dc44b363b7383dcd4fe, server=bd6592c37bb2,40087,1732486423713 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T22:16:46,408 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732486585339.8ca644d2cada6dc44b363b7383dcd4fe., hostname=bd6592c37bb2,40087,1732486423713, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8ca644d2cada6dc44b363b7383dcd4fe, server=bd6592c37bb2,40087,1732486423713 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T22:16:46,408 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40087 {}] ipc.CallRunner(138): callId: 149 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49060 deadline: 1732486666402, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8ca644d2cada6dc44b363b7383dcd4fe, server=bd6592c37bb2,40087,1732486423713 2024-11-24T22:16:46,408 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732486585339.8ca644d2cada6dc44b363b7383dcd4fe., hostname=bd6592c37bb2,40087,1732486423713, seqNum=5 because the exception is null or not the one we care about 2024-11-24T22:16:46,408 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732486585339.8ca644d2cada6dc44b363b7383dcd4fe., hostname=bd6592c37bb2,40087,1732486423713, seqNum=5 , the old value is region=TestAcidGuarantees,,1732486585339.8ca644d2cada6dc44b363b7383dcd4fe., hostname=bd6592c37bb2,40087,1732486423713, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8ca644d2cada6dc44b363b7383dcd4fe, server=bd6592c37bb2,40087,1732486423713 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T22:16:46,408 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732486585339.8ca644d2cada6dc44b363b7383dcd4fe., hostname=bd6592c37bb2,40087,1732486423713, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8ca644d2cada6dc44b363b7383dcd4fe, server=bd6592c37bb2,40087,1732486423713 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T22:16:46,408 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732486585339.8ca644d2cada6dc44b363b7383dcd4fe., hostname=bd6592c37bb2,40087,1732486423713, seqNum=5 because the exception is null or not the one we care about 2024-11-24T22:16:46,490 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37701 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=164 2024-11-24T22:16:46,531 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T22:16:46,534 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241124c28a62c49349432aaa77d1513ea0f9c4_8ca644d2cada6dc44b363b7383dcd4fe to hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241124c28a62c49349432aaa77d1513ea0f9c4_8ca644d2cada6dc44b363b7383dcd4fe 2024-11-24T22:16:46,535 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/8ca644d2cada6dc44b363b7383dcd4fe/.tmp/A/5b75c2f936b144c4a5e4bf896c7a3178, store: [table=TestAcidGuarantees family=A region=8ca644d2cada6dc44b363b7383dcd4fe] 2024-11-24T22:16:46,535 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/8ca644d2cada6dc44b363b7383dcd4fe/.tmp/A/5b75c2f936b144c4a5e4bf896c7a3178 is 175, key is test_row_0/A:col10/1732486606021/Put/seqid=0 2024-11-24T22:16:46,539 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45929 is added to blk_1073742465_1641 (size=39949) 2024-11-24T22:16:46,539 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=777, memsize=38.0 K, hasBloomFilter=true, into tmp file hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/8ca644d2cada6dc44b363b7383dcd4fe/.tmp/A/5b75c2f936b144c4a5e4bf896c7a3178 2024-11-24T22:16:46,541 INFO [RS:0;bd6592c37bb2:40087-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 8ca644d2cada6dc44b363b7383dcd4fe#A#compaction#536 average throughput is 0.05 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-24T22:16:46,542 DEBUG [RS:0;bd6592c37bb2:40087-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/8ca644d2cada6dc44b363b7383dcd4fe/.tmp/A/aca73ecfca3c475c865c0a6c969e035c is 175, key is test_row_0/A:col10/1732486604914/Put/seqid=0 2024-11-24T22:16:46,542 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=40087 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=165 2024-11-24T22:16:46,542 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bd6592c37bb2:0-1 {event_type=RS_FLUSH_REGIONS, pid=165}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732486585339.8ca644d2cada6dc44b363b7383dcd4fe. 2024-11-24T22:16:46,542 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bd6592c37bb2:0-1 {event_type=RS_FLUSH_REGIONS, pid=165}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732486585339.8ca644d2cada6dc44b363b7383dcd4fe. as already flushing 2024-11-24T22:16:46,542 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bd6592c37bb2:0-1 {event_type=RS_FLUSH_REGIONS, pid=165}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732486585339.8ca644d2cada6dc44b363b7383dcd4fe. 2024-11-24T22:16:46,542 ERROR [RS_FLUSH_OPERATIONS-regionserver/bd6592c37bb2:0-1 {event_type=RS_FLUSH_REGIONS, pid=165}] handler.RSProcedureHandler(58): pid=165 java.io.IOException: Unable to complete flush {ENCODED => 8ca644d2cada6dc44b363b7383dcd4fe, NAME => 'TestAcidGuarantees,,1732486585339.8ca644d2cada6dc44b363b7383dcd4fe.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T22:16:46,543 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bd6592c37bb2:0-1 {event_type=RS_FLUSH_REGIONS, pid=165}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=165 java.io.IOException: Unable to complete flush {ENCODED => 8ca644d2cada6dc44b363b7383dcd4fe, NAME => 'TestAcidGuarantees,,1732486585339.8ca644d2cada6dc44b363b7383dcd4fe.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T22:16:46,543 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37701 {}] master.HMaster(4177): Remote procedure failed, pid=165 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 8ca644d2cada6dc44b363b7383dcd4fe, NAME => 'TestAcidGuarantees,,1732486585339.8ca644d2cada6dc44b363b7383dcd4fe.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 8ca644d2cada6dc44b363b7383dcd4fe, NAME => 'TestAcidGuarantees,,1732486585339.8ca644d2cada6dc44b363b7383dcd4fe.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T22:16:46,546 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45929 is added to blk_1073742466_1642 (size=31903) 2024-11-24T22:16:46,547 DEBUG [RS:0;bd6592c37bb2:40087-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/8ca644d2cada6dc44b363b7383dcd4fe/.tmp/B/3b068b24fdb74c6190094f2727449194 as hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/8ca644d2cada6dc44b363b7383dcd4fe/B/3b068b24fdb74c6190094f2727449194 2024-11-24T22:16:46,549 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/8ca644d2cada6dc44b363b7383dcd4fe/.tmp/B/60a721145bed4ac1ac6c4ee15134ccdb is 50, key is test_row_0/B:col10/1732486606021/Put/seqid=0 2024-11-24T22:16:46,550 DEBUG [RS:0;bd6592c37bb2:40087-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/8ca644d2cada6dc44b363b7383dcd4fe/.tmp/A/aca73ecfca3c475c865c0a6c969e035c as hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/8ca644d2cada6dc44b363b7383dcd4fe/A/aca73ecfca3c475c865c0a6c969e035c 2024-11-24T22:16:46,552 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45929 is added to blk_1073742467_1643 (size=12301) 2024-11-24T22:16:46,554 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=38.02 KB at sequenceid=777 (bloomFilter=true), to=hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/8ca644d2cada6dc44b363b7383dcd4fe/.tmp/B/60a721145bed4ac1ac6c4ee15134ccdb 2024-11-24T22:16:46,554 INFO [RS:0;bd6592c37bb2:40087-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 8ca644d2cada6dc44b363b7383dcd4fe/B of 8ca644d2cada6dc44b363b7383dcd4fe into 3b068b24fdb74c6190094f2727449194(size=12.6 K), total size for store is 12.6 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-24T22:16:46,554 DEBUG [RS:0;bd6592c37bb2:40087-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 8ca644d2cada6dc44b363b7383dcd4fe: 2024-11-24T22:16:46,554 INFO [RS:0;bd6592c37bb2:40087-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732486585339.8ca644d2cada6dc44b363b7383dcd4fe., storeName=8ca644d2cada6dc44b363b7383dcd4fe/B, priority=12, startTime=1732486606052; duration=0sec 2024-11-24T22:16:46,554 DEBUG [RS:0;bd6592c37bb2:40087-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-24T22:16:46,554 DEBUG [RS:0;bd6592c37bb2:40087-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 8ca644d2cada6dc44b363b7383dcd4fe:B 2024-11-24T22:16:46,554 DEBUG [RS:0;bd6592c37bb2:40087-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-24T22:16:46,556 DEBUG [RS:0;bd6592c37bb2:40087-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 49566 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-24T22:16:46,556 DEBUG [RS:0;bd6592c37bb2:40087-longCompactions-0 {}] regionserver.HStore(1541): 8ca644d2cada6dc44b363b7383dcd4fe/C is initiating minor compaction (all files) 2024-11-24T22:16:46,556 INFO [RS:0;bd6592c37bb2:40087-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 8ca644d2cada6dc44b363b7383dcd4fe/C in TestAcidGuarantees,,1732486585339.8ca644d2cada6dc44b363b7383dcd4fe. 2024-11-24T22:16:46,556 INFO [RS:0;bd6592c37bb2:40087-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/8ca644d2cada6dc44b363b7383dcd4fe/C/ef06905113514929b387959c8588fb32, hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/8ca644d2cada6dc44b363b7383dcd4fe/C/943ba0f17f0c449e98e26c922ce03508, hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/8ca644d2cada6dc44b363b7383dcd4fe/C/05a2a98b5f2043a6a93bfa785195587f, hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/8ca644d2cada6dc44b363b7383dcd4fe/C/a88236e6fd224409ad07b616601f72ff] into tmpdir=hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/8ca644d2cada6dc44b363b7383dcd4fe/.tmp, totalSize=48.4 K 2024-11-24T22:16:46,556 DEBUG [RS:0;bd6592c37bb2:40087-longCompactions-0 {}] compactions.Compactor(225): Compacting ef06905113514929b387959c8588fb32, keycount=150, bloomtype=ROW, size=12.4 K, encoding=NONE, compression=NONE, seqNum=225, earliestPutTs=1732486601981 2024-11-24T22:16:46,556 DEBUG [RS:0;bd6592c37bb2:40087-longCompactions-0 {}] compactions.Compactor(225): Compacting 943ba0f17f0c449e98e26c922ce03508, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=695, earliestPutTs=1732486603614 2024-11-24T22:16:46,557 DEBUG [RS:0;bd6592c37bb2:40087-longCompactions-0 {}] compactions.Compactor(225): Compacting 05a2a98b5f2043a6a93bfa785195587f, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=733, earliestPutTs=1732486603755 2024-11-24T22:16:46,557 DEBUG [RS:0;bd6592c37bb2:40087-longCompactions-0 {}] compactions.Compactor(225): Compacting a88236e6fd224409ad07b616601f72ff, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=744, earliestPutTs=1732486604909 2024-11-24T22:16:46,557 INFO [RS:0;bd6592c37bb2:40087-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 8ca644d2cada6dc44b363b7383dcd4fe/A of 8ca644d2cada6dc44b363b7383dcd4fe into aca73ecfca3c475c865c0a6c969e035c(size=31.2 K), total size for store is 31.2 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-24T22:16:46,557 DEBUG [RS:0;bd6592c37bb2:40087-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 8ca644d2cada6dc44b363b7383dcd4fe: 2024-11-24T22:16:46,557 INFO [RS:0;bd6592c37bb2:40087-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732486585339.8ca644d2cada6dc44b363b7383dcd4fe., storeName=8ca644d2cada6dc44b363b7383dcd4fe/A, priority=12, startTime=1732486606052; duration=0sec 2024-11-24T22:16:46,557 DEBUG [RS:0;bd6592c37bb2:40087-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-24T22:16:46,557 DEBUG [RS:0;bd6592c37bb2:40087-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 8ca644d2cada6dc44b363b7383dcd4fe:A 2024-11-24T22:16:46,560 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/8ca644d2cada6dc44b363b7383dcd4fe/.tmp/C/d5f5e7c1ceb141538808728969043dc6 is 50, key is test_row_0/C:col10/1732486606021/Put/seqid=0 2024-11-24T22:16:46,562 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45929 is added to blk_1073742468_1644 (size=12301) 2024-11-24T22:16:46,564 INFO [RS:0;bd6592c37bb2:40087-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 8ca644d2cada6dc44b363b7383dcd4fe#C#compaction#541 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-24T22:16:46,565 DEBUG [RS:0;bd6592c37bb2:40087-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/8ca644d2cada6dc44b363b7383dcd4fe/.tmp/C/ae16909713a3403090f5a0ac150c170a is 50, key is test_row_0/C:col10/1732486604914/Put/seqid=0 2024-11-24T22:16:46,567 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45929 is added to blk_1073742469_1645 (size=12949) 2024-11-24T22:16:46,695 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=40087 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=165 2024-11-24T22:16:46,695 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bd6592c37bb2:0-2 {event_type=RS_FLUSH_REGIONS, pid=165}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732486585339.8ca644d2cada6dc44b363b7383dcd4fe. 2024-11-24T22:16:46,695 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bd6592c37bb2:0-2 {event_type=RS_FLUSH_REGIONS, pid=165}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732486585339.8ca644d2cada6dc44b363b7383dcd4fe. as already flushing 2024-11-24T22:16:46,695 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bd6592c37bb2:0-2 {event_type=RS_FLUSH_REGIONS, pid=165}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732486585339.8ca644d2cada6dc44b363b7383dcd4fe. 2024-11-24T22:16:46,695 ERROR [RS_FLUSH_OPERATIONS-regionserver/bd6592c37bb2:0-2 {event_type=RS_FLUSH_REGIONS, pid=165}] handler.RSProcedureHandler(58): pid=165 java.io.IOException: Unable to complete flush {ENCODED => 8ca644d2cada6dc44b363b7383dcd4fe, NAME => 'TestAcidGuarantees,,1732486585339.8ca644d2cada6dc44b363b7383dcd4fe.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T22:16:46,696 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bd6592c37bb2:0-2 {event_type=RS_FLUSH_REGIONS, pid=165}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=165 java.io.IOException: Unable to complete flush {ENCODED => 8ca644d2cada6dc44b363b7383dcd4fe, NAME => 'TestAcidGuarantees,,1732486585339.8ca644d2cada6dc44b363b7383dcd4fe.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T22:16:46,696 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37701 {}] master.HMaster(4177): Remote procedure failed, pid=165 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 8ca644d2cada6dc44b363b7383dcd4fe, NAME => 'TestAcidGuarantees,,1732486585339.8ca644d2cada6dc44b363b7383dcd4fe.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 8ca644d2cada6dc44b363b7383dcd4fe, NAME => 'TestAcidGuarantees,,1732486585339.8ca644d2cada6dc44b363b7383dcd4fe.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T22:16:46,701 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37701 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=164 2024-11-24T22:16:46,715 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40087 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8ca644d2cada6dc44b363b7383dcd4fe, server=bd6592c37bb2,40087,1732486423713 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-24T22:16:46,715 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40087 {}] ipc.CallRunner(138): callId: 234 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49050 deadline: 1732486666711, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8ca644d2cada6dc44b363b7383dcd4fe, server=bd6592c37bb2,40087,1732486423713 2024-11-24T22:16:46,715 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732486585339.8ca644d2cada6dc44b363b7383dcd4fe., hostname=bd6592c37bb2,40087,1732486423713, seqNum=5 , the old value is region=TestAcidGuarantees,,1732486585339.8ca644d2cada6dc44b363b7383dcd4fe., hostname=bd6592c37bb2,40087,1732486423713, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8ca644d2cada6dc44b363b7383dcd4fe, server=bd6592c37bb2,40087,1732486423713 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T22:16:46,715 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732486585339.8ca644d2cada6dc44b363b7383dcd4fe., hostname=bd6592c37bb2,40087,1732486423713, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8ca644d2cada6dc44b363b7383dcd4fe, server=bd6592c37bb2,40087,1732486423713 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T22:16:46,715 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732486585339.8ca644d2cada6dc44b363b7383dcd4fe., hostname=bd6592c37bb2,40087,1732486423713, seqNum=5 because the exception is null or not the one we care about 2024-11-24T22:16:46,716 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40087 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8ca644d2cada6dc44b363b7383dcd4fe, server=bd6592c37bb2,40087,1732486423713 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-24T22:16:46,716 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40087 {}] ipc.CallRunner(138): callId: 208 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48986 deadline: 1732486666711, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8ca644d2cada6dc44b363b7383dcd4fe, server=bd6592c37bb2,40087,1732486423713 2024-11-24T22:16:46,716 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732486585339.8ca644d2cada6dc44b363b7383dcd4fe., hostname=bd6592c37bb2,40087,1732486423713, seqNum=5 , the old value is region=TestAcidGuarantees,,1732486585339.8ca644d2cada6dc44b363b7383dcd4fe., hostname=bd6592c37bb2,40087,1732486423713, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8ca644d2cada6dc44b363b7383dcd4fe, server=bd6592c37bb2,40087,1732486423713 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T22:16:46,716 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40087 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8ca644d2cada6dc44b363b7383dcd4fe, server=bd6592c37bb2,40087,1732486423713 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-24T22:16:46,716 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732486585339.8ca644d2cada6dc44b363b7383dcd4fe., hostname=bd6592c37bb2,40087,1732486423713, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8ca644d2cada6dc44b363b7383dcd4fe, server=bd6592c37bb2,40087,1732486423713 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T22:16:46,716 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732486585339.8ca644d2cada6dc44b363b7383dcd4fe., hostname=bd6592c37bb2,40087,1732486423713, seqNum=5 because the exception is null or not the one we care about 2024-11-24T22:16:46,716 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40087 {}] ipc.CallRunner(138): callId: 150 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49060 deadline: 1732486666711, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8ca644d2cada6dc44b363b7383dcd4fe, server=bd6592c37bb2,40087,1732486423713 2024-11-24T22:16:46,716 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732486585339.8ca644d2cada6dc44b363b7383dcd4fe., hostname=bd6592c37bb2,40087,1732486423713, seqNum=5 , the old value is region=TestAcidGuarantees,,1732486585339.8ca644d2cada6dc44b363b7383dcd4fe., hostname=bd6592c37bb2,40087,1732486423713, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8ca644d2cada6dc44b363b7383dcd4fe, server=bd6592c37bb2,40087,1732486423713 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T22:16:46,716 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732486585339.8ca644d2cada6dc44b363b7383dcd4fe., hostname=bd6592c37bb2,40087,1732486423713, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8ca644d2cada6dc44b363b7383dcd4fe, server=bd6592c37bb2,40087,1732486423713 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T22:16:46,716 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732486585339.8ca644d2cada6dc44b363b7383dcd4fe., hostname=bd6592c37bb2,40087,1732486423713, seqNum=5 because the exception is null or not the one we care about 2024-11-24T22:16:46,724 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40087 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8ca644d2cada6dc44b363b7383dcd4fe, server=bd6592c37bb2,40087,1732486423713 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-24T22:16:46,725 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40087 {}] ipc.CallRunner(138): callId: 125 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49026 deadline: 1732486666720, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8ca644d2cada6dc44b363b7383dcd4fe, server=bd6592c37bb2,40087,1732486423713 2024-11-24T22:16:46,725 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732486585339.8ca644d2cada6dc44b363b7383dcd4fe., hostname=bd6592c37bb2,40087,1732486423713, seqNum=5 , the old value is region=TestAcidGuarantees,,1732486585339.8ca644d2cada6dc44b363b7383dcd4fe., hostname=bd6592c37bb2,40087,1732486423713, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8ca644d2cada6dc44b363b7383dcd4fe, server=bd6592c37bb2,40087,1732486423713 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T22:16:46,725 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732486585339.8ca644d2cada6dc44b363b7383dcd4fe., hostname=bd6592c37bb2,40087,1732486423713, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8ca644d2cada6dc44b363b7383dcd4fe, server=bd6592c37bb2,40087,1732486423713 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T22:16:46,725 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732486585339.8ca644d2cada6dc44b363b7383dcd4fe., hostname=bd6592c37bb2,40087,1732486423713, seqNum=5 because the exception is null or not the one we care about 2024-11-24T22:16:46,725 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40087 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8ca644d2cada6dc44b363b7383dcd4fe, server=bd6592c37bb2,40087,1732486423713 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-24T22:16:46,725 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40087 {}] ipc.CallRunner(138): callId: 199 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49002 deadline: 1732486666720, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8ca644d2cada6dc44b363b7383dcd4fe, server=bd6592c37bb2,40087,1732486423713 2024-11-24T22:16:46,725 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732486585339.8ca644d2cada6dc44b363b7383dcd4fe., hostname=bd6592c37bb2,40087,1732486423713, seqNum=5 , the old value is region=TestAcidGuarantees,,1732486585339.8ca644d2cada6dc44b363b7383dcd4fe., hostname=bd6592c37bb2,40087,1732486423713, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8ca644d2cada6dc44b363b7383dcd4fe, server=bd6592c37bb2,40087,1732486423713 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T22:16:46,725 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732486585339.8ca644d2cada6dc44b363b7383dcd4fe., hostname=bd6592c37bb2,40087,1732486423713, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8ca644d2cada6dc44b363b7383dcd4fe, server=bd6592c37bb2,40087,1732486423713 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T22:16:46,725 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732486585339.8ca644d2cada6dc44b363b7383dcd4fe., hostname=bd6592c37bb2,40087,1732486423713, seqNum=5 because the exception is null or not the one we care about 2024-11-24T22:16:46,848 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=40087 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=165 2024-11-24T22:16:46,848 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bd6592c37bb2:0-0 {event_type=RS_FLUSH_REGIONS, pid=165}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732486585339.8ca644d2cada6dc44b363b7383dcd4fe. 2024-11-24T22:16:46,848 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bd6592c37bb2:0-0 {event_type=RS_FLUSH_REGIONS, pid=165}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732486585339.8ca644d2cada6dc44b363b7383dcd4fe. as already flushing 2024-11-24T22:16:46,848 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bd6592c37bb2:0-0 {event_type=RS_FLUSH_REGIONS, pid=165}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732486585339.8ca644d2cada6dc44b363b7383dcd4fe. 2024-11-24T22:16:46,848 ERROR [RS_FLUSH_OPERATIONS-regionserver/bd6592c37bb2:0-0 {event_type=RS_FLUSH_REGIONS, pid=165}] handler.RSProcedureHandler(58): pid=165 java.io.IOException: Unable to complete flush {ENCODED => 8ca644d2cada6dc44b363b7383dcd4fe, NAME => 'TestAcidGuarantees,,1732486585339.8ca644d2cada6dc44b363b7383dcd4fe.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T22:16:46,848 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bd6592c37bb2:0-0 {event_type=RS_FLUSH_REGIONS, pid=165}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=165 java.io.IOException: Unable to complete flush {ENCODED => 8ca644d2cada6dc44b363b7383dcd4fe, NAME => 'TestAcidGuarantees,,1732486585339.8ca644d2cada6dc44b363b7383dcd4fe.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T22:16:46,849 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37701 {}] master.HMaster(4177): Remote procedure failed, pid=165 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 8ca644d2cada6dc44b363b7383dcd4fe, NAME => 'TestAcidGuarantees,,1732486585339.8ca644d2cada6dc44b363b7383dcd4fe.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 8ca644d2cada6dc44b363b7383dcd4fe, NAME => 'TestAcidGuarantees,,1732486585339.8ca644d2cada6dc44b363b7383dcd4fe.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T22:16:46,963 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=35.78 KB at sequenceid=777 (bloomFilter=true), to=hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/8ca644d2cada6dc44b363b7383dcd4fe/.tmp/C/d5f5e7c1ceb141538808728969043dc6 2024-11-24T22:16:46,967 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/8ca644d2cada6dc44b363b7383dcd4fe/.tmp/A/5b75c2f936b144c4a5e4bf896c7a3178 as hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/8ca644d2cada6dc44b363b7383dcd4fe/A/5b75c2f936b144c4a5e4bf896c7a3178 2024-11-24T22:16:46,970 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/8ca644d2cada6dc44b363b7383dcd4fe/A/5b75c2f936b144c4a5e4bf896c7a3178, entries=200, sequenceid=777, filesize=39.0 K 2024-11-24T22:16:46,971 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/8ca644d2cada6dc44b363b7383dcd4fe/.tmp/B/60a721145bed4ac1ac6c4ee15134ccdb as hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/8ca644d2cada6dc44b363b7383dcd4fe/B/60a721145bed4ac1ac6c4ee15134ccdb 2024-11-24T22:16:46,971 DEBUG [RS:0;bd6592c37bb2:40087-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/8ca644d2cada6dc44b363b7383dcd4fe/.tmp/C/ae16909713a3403090f5a0ac150c170a as hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/8ca644d2cada6dc44b363b7383dcd4fe/C/ae16909713a3403090f5a0ac150c170a 2024-11-24T22:16:46,974 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/8ca644d2cada6dc44b363b7383dcd4fe/B/60a721145bed4ac1ac6c4ee15134ccdb, entries=150, sequenceid=777, filesize=12.0 K 2024-11-24T22:16:46,975 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/8ca644d2cada6dc44b363b7383dcd4fe/.tmp/C/d5f5e7c1ceb141538808728969043dc6 as hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/8ca644d2cada6dc44b363b7383dcd4fe/C/d5f5e7c1ceb141538808728969043dc6 2024-11-24T22:16:46,975 INFO [RS:0;bd6592c37bb2:40087-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 8ca644d2cada6dc44b363b7383dcd4fe/C of 8ca644d2cada6dc44b363b7383dcd4fe into ae16909713a3403090f5a0ac150c170a(size=12.6 K), total size for store is 12.6 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-24T22:16:46,975 DEBUG [RS:0;bd6592c37bb2:40087-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 8ca644d2cada6dc44b363b7383dcd4fe: 2024-11-24T22:16:46,975 INFO [RS:0;bd6592c37bb2:40087-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732486585339.8ca644d2cada6dc44b363b7383dcd4fe., storeName=8ca644d2cada6dc44b363b7383dcd4fe/C, priority=12, startTime=1732486606052; duration=0sec 2024-11-24T22:16:46,975 DEBUG [RS:0;bd6592c37bb2:40087-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-24T22:16:46,975 DEBUG [RS:0;bd6592c37bb2:40087-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 8ca644d2cada6dc44b363b7383dcd4fe:C 2024-11-24T22:16:46,977 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/8ca644d2cada6dc44b363b7383dcd4fe/C/d5f5e7c1ceb141538808728969043dc6, entries=150, sequenceid=777, filesize=12.0 K 2024-11-24T22:16:46,979 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~111.82 KB/114500, heapSize ~315.41 KB/322976, currentSize=67.09 KB/68700 for 8ca644d2cada6dc44b363b7383dcd4fe in 926ms, sequenceid=777, compaction requested=false 2024-11-24T22:16:46,979 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 8ca644d2cada6dc44b363b7383dcd4fe: 2024-11-24T22:16:47,001 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=40087 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=165 2024-11-24T22:16:47,001 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bd6592c37bb2:0-1 {event_type=RS_FLUSH_REGIONS, pid=165}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732486585339.8ca644d2cada6dc44b363b7383dcd4fe. 2024-11-24T22:16:47,001 INFO [RS_FLUSH_OPERATIONS-regionserver/bd6592c37bb2:0-1 {event_type=RS_FLUSH_REGIONS, pid=165}] regionserver.HRegion(2902): Flushing 8ca644d2cada6dc44b363b7383dcd4fe 3/3 column families, dataSize=67.09 KB heapSize=199.97 KB 2024-11-24T22:16:47,001 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bd6592c37bb2:0-1 {event_type=RS_FLUSH_REGIONS, pid=165}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 8ca644d2cada6dc44b363b7383dcd4fe, store=A 2024-11-24T22:16:47,001 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bd6592c37bb2:0-1 {event_type=RS_FLUSH_REGIONS, pid=165}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-24T22:16:47,001 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bd6592c37bb2:0-1 {event_type=RS_FLUSH_REGIONS, pid=165}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 8ca644d2cada6dc44b363b7383dcd4fe, store=B 2024-11-24T22:16:47,001 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bd6592c37bb2:0-1 {event_type=RS_FLUSH_REGIONS, pid=165}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-24T22:16:47,001 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bd6592c37bb2:0-1 {event_type=RS_FLUSH_REGIONS, pid=165}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 8ca644d2cada6dc44b363b7383dcd4fe, store=C 2024-11-24T22:16:47,001 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bd6592c37bb2:0-1 {event_type=RS_FLUSH_REGIONS, pid=165}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-24T22:16:47,008 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bd6592c37bb2:0-1 {event_type=RS_FLUSH_REGIONS, pid=165}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241124e959a70aeeb4453c80fd9a09da9e639b_8ca644d2cada6dc44b363b7383dcd4fe is 50, key is test_row_0/A:col10/1732486606070/Put/seqid=0 2024-11-24T22:16:47,011 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37701 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=164 2024-11-24T22:16:47,011 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45929 is added to blk_1073742470_1646 (size=12454) 2024-11-24T22:16:47,012 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bd6592c37bb2:0-1 {event_type=RS_FLUSH_REGIONS, pid=165}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T22:16:47,014 INFO [RS_FLUSH_OPERATIONS-regionserver/bd6592c37bb2:0-1 {event_type=RS_FLUSH_REGIONS, pid=165}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241124e959a70aeeb4453c80fd9a09da9e639b_8ca644d2cada6dc44b363b7383dcd4fe to hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241124e959a70aeeb4453c80fd9a09da9e639b_8ca644d2cada6dc44b363b7383dcd4fe 2024-11-24T22:16:47,015 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bd6592c37bb2:0-1 {event_type=RS_FLUSH_REGIONS, pid=165}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/8ca644d2cada6dc44b363b7383dcd4fe/.tmp/A/edcdc9c2d1544300b712df3846855e54, store: [table=TestAcidGuarantees family=A region=8ca644d2cada6dc44b363b7383dcd4fe] 2024-11-24T22:16:47,016 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bd6592c37bb2:0-1 {event_type=RS_FLUSH_REGIONS, pid=165}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/8ca644d2cada6dc44b363b7383dcd4fe/.tmp/A/edcdc9c2d1544300b712df3846855e54 is 175, key is test_row_0/A:col10/1732486606070/Put/seqid=0 2024-11-24T22:16:47,018 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45929 is added to blk_1073742471_1647 (size=31255) 2024-11-24T22:16:47,223 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40087 {}] regionserver.HRegion(8855): Flush requested on 8ca644d2cada6dc44b363b7383dcd4fe 2024-11-24T22:16:47,223 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732486585339.8ca644d2cada6dc44b363b7383dcd4fe. as already flushing 2024-11-24T22:16:47,249 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40087 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8ca644d2cada6dc44b363b7383dcd4fe, server=bd6592c37bb2,40087,1732486423713 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-24T22:16:47,249 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40087 {}] ipc.CallRunner(138): callId: 211 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48986 deadline: 1732486667243, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8ca644d2cada6dc44b363b7383dcd4fe, server=bd6592c37bb2,40087,1732486423713 2024-11-24T22:16:47,249 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732486585339.8ca644d2cada6dc44b363b7383dcd4fe., hostname=bd6592c37bb2,40087,1732486423713, seqNum=5 , the old value is region=TestAcidGuarantees,,1732486585339.8ca644d2cada6dc44b363b7383dcd4fe., hostname=bd6592c37bb2,40087,1732486423713, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8ca644d2cada6dc44b363b7383dcd4fe, server=bd6592c37bb2,40087,1732486423713 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T22:16:47,249 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732486585339.8ca644d2cada6dc44b363b7383dcd4fe., hostname=bd6592c37bb2,40087,1732486423713, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8ca644d2cada6dc44b363b7383dcd4fe, server=bd6592c37bb2,40087,1732486423713 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T22:16:47,249 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732486585339.8ca644d2cada6dc44b363b7383dcd4fe., hostname=bd6592c37bb2,40087,1732486423713, seqNum=5 because the exception is null or not the one we care about 2024-11-24T22:16:47,251 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40087 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8ca644d2cada6dc44b363b7383dcd4fe, server=bd6592c37bb2,40087,1732486423713 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-24T22:16:47,251 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40087 {}] ipc.CallRunner(138): callId: 128 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49026 deadline: 1732486667245, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8ca644d2cada6dc44b363b7383dcd4fe, server=bd6592c37bb2,40087,1732486423713 2024-11-24T22:16:47,252 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40087 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8ca644d2cada6dc44b363b7383dcd4fe, server=bd6592c37bb2,40087,1732486423713 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-24T22:16:47,252 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40087 {}] ipc.CallRunner(138): callId: 202 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49002 deadline: 1732486667245, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8ca644d2cada6dc44b363b7383dcd4fe, server=bd6592c37bb2,40087,1732486423713 2024-11-24T22:16:47,252 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732486585339.8ca644d2cada6dc44b363b7383dcd4fe., hostname=bd6592c37bb2,40087,1732486423713, seqNum=5 , the old value is region=TestAcidGuarantees,,1732486585339.8ca644d2cada6dc44b363b7383dcd4fe., hostname=bd6592c37bb2,40087,1732486423713, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8ca644d2cada6dc44b363b7383dcd4fe, server=bd6592c37bb2,40087,1732486423713 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T22:16:47,252 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732486585339.8ca644d2cada6dc44b363b7383dcd4fe., hostname=bd6592c37bb2,40087,1732486423713, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8ca644d2cada6dc44b363b7383dcd4fe, server=bd6592c37bb2,40087,1732486423713 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T22:16:47,252 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732486585339.8ca644d2cada6dc44b363b7383dcd4fe., hostname=bd6592c37bb2,40087,1732486423713, seqNum=5 because the exception is null or not the one we care about 2024-11-24T22:16:47,252 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40087 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8ca644d2cada6dc44b363b7383dcd4fe, server=bd6592c37bb2,40087,1732486423713 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-24T22:16:47,252 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732486585339.8ca644d2cada6dc44b363b7383dcd4fe., hostname=bd6592c37bb2,40087,1732486423713, seqNum=5 , the old value is region=TestAcidGuarantees,,1732486585339.8ca644d2cada6dc44b363b7383dcd4fe., hostname=bd6592c37bb2,40087,1732486423713, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8ca644d2cada6dc44b363b7383dcd4fe, server=bd6592c37bb2,40087,1732486423713 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T22:16:47,252 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732486585339.8ca644d2cada6dc44b363b7383dcd4fe., hostname=bd6592c37bb2,40087,1732486423713, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8ca644d2cada6dc44b363b7383dcd4fe, server=bd6592c37bb2,40087,1732486423713 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T22:16:47,252 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40087 {}] ipc.CallRunner(138): callId: 240 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49050 deadline: 1732486667247, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8ca644d2cada6dc44b363b7383dcd4fe, server=bd6592c37bb2,40087,1732486423713 2024-11-24T22:16:47,252 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732486585339.8ca644d2cada6dc44b363b7383dcd4fe., hostname=bd6592c37bb2,40087,1732486423713, seqNum=5 because the exception is null or not the one we care about 2024-11-24T22:16:47,252 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40087 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8ca644d2cada6dc44b363b7383dcd4fe, server=bd6592c37bb2,40087,1732486423713 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-24T22:16:47,252 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732486585339.8ca644d2cada6dc44b363b7383dcd4fe., hostname=bd6592c37bb2,40087,1732486423713, seqNum=5 , the old value is region=TestAcidGuarantees,,1732486585339.8ca644d2cada6dc44b363b7383dcd4fe., hostname=bd6592c37bb2,40087,1732486423713, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8ca644d2cada6dc44b363b7383dcd4fe, server=bd6592c37bb2,40087,1732486423713 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T22:16:47,252 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732486585339.8ca644d2cada6dc44b363b7383dcd4fe., hostname=bd6592c37bb2,40087,1732486423713, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8ca644d2cada6dc44b363b7383dcd4fe, server=bd6592c37bb2,40087,1732486423713 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T22:16:47,252 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40087 {}] ipc.CallRunner(138): callId: 156 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49060 deadline: 1732486667247, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8ca644d2cada6dc44b363b7383dcd4fe, server=bd6592c37bb2,40087,1732486423713 2024-11-24T22:16:47,252 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732486585339.8ca644d2cada6dc44b363b7383dcd4fe., hostname=bd6592c37bb2,40087,1732486423713, seqNum=5 because the exception is null or not the one we care about 2024-11-24T22:16:47,253 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732486585339.8ca644d2cada6dc44b363b7383dcd4fe., hostname=bd6592c37bb2,40087,1732486423713, seqNum=5 , the old value is region=TestAcidGuarantees,,1732486585339.8ca644d2cada6dc44b363b7383dcd4fe., hostname=bd6592c37bb2,40087,1732486423713, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8ca644d2cada6dc44b363b7383dcd4fe, server=bd6592c37bb2,40087,1732486423713 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T22:16:47,253 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732486585339.8ca644d2cada6dc44b363b7383dcd4fe., hostname=bd6592c37bb2,40087,1732486423713, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8ca644d2cada6dc44b363b7383dcd4fe, server=bd6592c37bb2,40087,1732486423713 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T22:16:47,253 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732486585339.8ca644d2cada6dc44b363b7383dcd4fe., hostname=bd6592c37bb2,40087,1732486423713, seqNum=5 because the exception is null or not the one we care about 2024-11-24T22:16:47,364 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40087 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8ca644d2cada6dc44b363b7383dcd4fe, server=bd6592c37bb2,40087,1732486423713 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-24T22:16:47,364 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40087 {}] ipc.CallRunner(138): callId: 212 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48986 deadline: 1732486667360, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8ca644d2cada6dc44b363b7383dcd4fe, server=bd6592c37bb2,40087,1732486423713 2024-11-24T22:16:47,365 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40087 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8ca644d2cada6dc44b363b7383dcd4fe, server=bd6592c37bb2,40087,1732486423713 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-24T22:16:47,365 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732486585339.8ca644d2cada6dc44b363b7383dcd4fe., hostname=bd6592c37bb2,40087,1732486423713, seqNum=5 , the old value is region=TestAcidGuarantees,,1732486585339.8ca644d2cada6dc44b363b7383dcd4fe., hostname=bd6592c37bb2,40087,1732486423713, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8ca644d2cada6dc44b363b7383dcd4fe, server=bd6592c37bb2,40087,1732486423713 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T22:16:47,365 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40087 {}] ipc.CallRunner(138): callId: 129 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49026 deadline: 1732486667360, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8ca644d2cada6dc44b363b7383dcd4fe, server=bd6592c37bb2,40087,1732486423713 2024-11-24T22:16:47,365 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732486585339.8ca644d2cada6dc44b363b7383dcd4fe., hostname=bd6592c37bb2,40087,1732486423713, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8ca644d2cada6dc44b363b7383dcd4fe, server=bd6592c37bb2,40087,1732486423713 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T22:16:47,365 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732486585339.8ca644d2cada6dc44b363b7383dcd4fe., hostname=bd6592c37bb2,40087,1732486423713, seqNum=5 because the exception is null or not the one we care about 2024-11-24T22:16:47,365 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40087 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8ca644d2cada6dc44b363b7383dcd4fe, server=bd6592c37bb2,40087,1732486423713 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-24T22:16:47,365 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732486585339.8ca644d2cada6dc44b363b7383dcd4fe., hostname=bd6592c37bb2,40087,1732486423713, seqNum=5 , the old value is region=TestAcidGuarantees,,1732486585339.8ca644d2cada6dc44b363b7383dcd4fe., hostname=bd6592c37bb2,40087,1732486423713, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8ca644d2cada6dc44b363b7383dcd4fe, server=bd6592c37bb2,40087,1732486423713 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T22:16:47,365 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732486585339.8ca644d2cada6dc44b363b7383dcd4fe., hostname=bd6592c37bb2,40087,1732486423713, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8ca644d2cada6dc44b363b7383dcd4fe, server=bd6592c37bb2,40087,1732486423713 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T22:16:47,365 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40087 {}] ipc.CallRunner(138): callId: 203 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49002 deadline: 1732486667361, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8ca644d2cada6dc44b363b7383dcd4fe, server=bd6592c37bb2,40087,1732486423713 2024-11-24T22:16:47,365 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732486585339.8ca644d2cada6dc44b363b7383dcd4fe., hostname=bd6592c37bb2,40087,1732486423713, seqNum=5 because the exception is null or not the one we care about 2024-11-24T22:16:47,365 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732486585339.8ca644d2cada6dc44b363b7383dcd4fe., hostname=bd6592c37bb2,40087,1732486423713, seqNum=5 , the old value is region=TestAcidGuarantees,,1732486585339.8ca644d2cada6dc44b363b7383dcd4fe., hostname=bd6592c37bb2,40087,1732486423713, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8ca644d2cada6dc44b363b7383dcd4fe, server=bd6592c37bb2,40087,1732486423713 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T22:16:47,365 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732486585339.8ca644d2cada6dc44b363b7383dcd4fe., hostname=bd6592c37bb2,40087,1732486423713, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8ca644d2cada6dc44b363b7383dcd4fe, server=bd6592c37bb2,40087,1732486423713 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T22:16:47,365 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40087 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8ca644d2cada6dc44b363b7383dcd4fe, server=bd6592c37bb2,40087,1732486423713 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-24T22:16:47,365 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732486585339.8ca644d2cada6dc44b363b7383dcd4fe., hostname=bd6592c37bb2,40087,1732486423713, seqNum=5 because the exception is null or not the one we care about 2024-11-24T22:16:47,365 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40087 {}] ipc.CallRunner(138): callId: 241 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49050 deadline: 1732486667361, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8ca644d2cada6dc44b363b7383dcd4fe, server=bd6592c37bb2,40087,1732486423713 2024-11-24T22:16:47,366 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40087 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8ca644d2cada6dc44b363b7383dcd4fe, server=bd6592c37bb2,40087,1732486423713 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-24T22:16:47,366 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732486585339.8ca644d2cada6dc44b363b7383dcd4fe., hostname=bd6592c37bb2,40087,1732486423713, seqNum=5 , the old value is region=TestAcidGuarantees,,1732486585339.8ca644d2cada6dc44b363b7383dcd4fe., hostname=bd6592c37bb2,40087,1732486423713, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8ca644d2cada6dc44b363b7383dcd4fe, server=bd6592c37bb2,40087,1732486423713 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T22:16:47,366 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732486585339.8ca644d2cada6dc44b363b7383dcd4fe., hostname=bd6592c37bb2,40087,1732486423713, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8ca644d2cada6dc44b363b7383dcd4fe, server=bd6592c37bb2,40087,1732486423713 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T22:16:47,366 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40087 {}] ipc.CallRunner(138): callId: 157 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49060 deadline: 1732486667361, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8ca644d2cada6dc44b363b7383dcd4fe, server=bd6592c37bb2,40087,1732486423713 2024-11-24T22:16:47,366 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732486585339.8ca644d2cada6dc44b363b7383dcd4fe., hostname=bd6592c37bb2,40087,1732486423713, seqNum=5 because the exception is null or not the one we care about 2024-11-24T22:16:47,366 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732486585339.8ca644d2cada6dc44b363b7383dcd4fe., hostname=bd6592c37bb2,40087,1732486423713, seqNum=5 , the old value is region=TestAcidGuarantees,,1732486585339.8ca644d2cada6dc44b363b7383dcd4fe., hostname=bd6592c37bb2,40087,1732486423713, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8ca644d2cada6dc44b363b7383dcd4fe, server=bd6592c37bb2,40087,1732486423713 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T22:16:47,366 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732486585339.8ca644d2cada6dc44b363b7383dcd4fe., hostname=bd6592c37bb2,40087,1732486423713, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8ca644d2cada6dc44b363b7383dcd4fe, server=bd6592c37bb2,40087,1732486423713 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T22:16:47,366 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732486585339.8ca644d2cada6dc44b363b7383dcd4fe., hostname=bd6592c37bb2,40087,1732486423713, seqNum=5 because the exception is null or not the one we care about 2024-11-24T22:16:47,419 INFO [RS_FLUSH_OPERATIONS-regionserver/bd6592c37bb2:0-1 {event_type=RS_FLUSH_REGIONS, pid=165}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=793, memsize=22.4 K, hasBloomFilter=true, into tmp file hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/8ca644d2cada6dc44b363b7383dcd4fe/.tmp/A/edcdc9c2d1544300b712df3846855e54 2024-11-24T22:16:47,426 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bd6592c37bb2:0-1 {event_type=RS_FLUSH_REGIONS, pid=165}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/8ca644d2cada6dc44b363b7383dcd4fe/.tmp/B/8b5d192284234dd797b73ec2a04209f8 is 50, key is test_row_0/B:col10/1732486606070/Put/seqid=0 2024-11-24T22:16:47,430 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45929 is added to blk_1073742472_1648 (size=12301) 2024-11-24T22:16:47,430 INFO [RS_FLUSH_OPERATIONS-regionserver/bd6592c37bb2:0-1 {event_type=RS_FLUSH_REGIONS, pid=165}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=793 (bloomFilter=true), to=hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/8ca644d2cada6dc44b363b7383dcd4fe/.tmp/B/8b5d192284234dd797b73ec2a04209f8 2024-11-24T22:16:47,437 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bd6592c37bb2:0-1 {event_type=RS_FLUSH_REGIONS, pid=165}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/8ca644d2cada6dc44b363b7383dcd4fe/.tmp/C/2795ef0eb8634c1a81eabaac53d9372a is 50, key is test_row_0/C:col10/1732486606070/Put/seqid=0 2024-11-24T22:16:47,440 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45929 is added to blk_1073742473_1649 (size=12301) 2024-11-24T22:16:47,441 INFO [RS_FLUSH_OPERATIONS-regionserver/bd6592c37bb2:0-1 {event_type=RS_FLUSH_REGIONS, pid=165}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=793 (bloomFilter=true), to=hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/8ca644d2cada6dc44b363b7383dcd4fe/.tmp/C/2795ef0eb8634c1a81eabaac53d9372a 2024-11-24T22:16:47,443 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bd6592c37bb2:0-1 {event_type=RS_FLUSH_REGIONS, pid=165}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/8ca644d2cada6dc44b363b7383dcd4fe/.tmp/A/edcdc9c2d1544300b712df3846855e54 as hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/8ca644d2cada6dc44b363b7383dcd4fe/A/edcdc9c2d1544300b712df3846855e54 2024-11-24T22:16:47,447 INFO [RS_FLUSH_OPERATIONS-regionserver/bd6592c37bb2:0-1 {event_type=RS_FLUSH_REGIONS, pid=165}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/8ca644d2cada6dc44b363b7383dcd4fe/A/edcdc9c2d1544300b712df3846855e54, entries=150, sequenceid=793, filesize=30.5 K 2024-11-24T22:16:47,448 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bd6592c37bb2:0-1 {event_type=RS_FLUSH_REGIONS, pid=165}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/8ca644d2cada6dc44b363b7383dcd4fe/.tmp/B/8b5d192284234dd797b73ec2a04209f8 as hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/8ca644d2cada6dc44b363b7383dcd4fe/B/8b5d192284234dd797b73ec2a04209f8 2024-11-24T22:16:47,451 INFO [RS_FLUSH_OPERATIONS-regionserver/bd6592c37bb2:0-1 {event_type=RS_FLUSH_REGIONS, pid=165}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/8ca644d2cada6dc44b363b7383dcd4fe/B/8b5d192284234dd797b73ec2a04209f8, entries=150, sequenceid=793, filesize=12.0 K 2024-11-24T22:16:47,451 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bd6592c37bb2:0-1 {event_type=RS_FLUSH_REGIONS, pid=165}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/8ca644d2cada6dc44b363b7383dcd4fe/.tmp/C/2795ef0eb8634c1a81eabaac53d9372a as hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/8ca644d2cada6dc44b363b7383dcd4fe/C/2795ef0eb8634c1a81eabaac53d9372a 2024-11-24T22:16:47,454 INFO [RS_FLUSH_OPERATIONS-regionserver/bd6592c37bb2:0-1 {event_type=RS_FLUSH_REGIONS, pid=165}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/8ca644d2cada6dc44b363b7383dcd4fe/C/2795ef0eb8634c1a81eabaac53d9372a, entries=150, sequenceid=793, filesize=12.0 K 2024-11-24T22:16:47,454 INFO [RS_FLUSH_OPERATIONS-regionserver/bd6592c37bb2:0-1 {event_type=RS_FLUSH_REGIONS, pid=165}] regionserver.HRegion(3140): Finished flush of dataSize ~67.09 KB/68700, heapSize ~199.92 KB/204720, currentSize=107.34 KB/109920 for 8ca644d2cada6dc44b363b7383dcd4fe in 453ms, sequenceid=793, compaction requested=true 2024-11-24T22:16:47,454 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bd6592c37bb2:0-1 {event_type=RS_FLUSH_REGIONS, pid=165}] regionserver.HRegion(2603): Flush status journal for 8ca644d2cada6dc44b363b7383dcd4fe: 2024-11-24T22:16:47,454 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bd6592c37bb2:0-1 {event_type=RS_FLUSH_REGIONS, pid=165}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732486585339.8ca644d2cada6dc44b363b7383dcd4fe. 2024-11-24T22:16:47,455 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bd6592c37bb2:0-1 {event_type=RS_FLUSH_REGIONS, pid=165}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=165 2024-11-24T22:16:47,455 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37701 {}] master.HMaster(4169): Remote procedure done, pid=165 2024-11-24T22:16:47,457 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=165, resume processing ppid=164 2024-11-24T22:16:47,457 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=165, ppid=164, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.0660 sec 2024-11-24T22:16:47,458 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=164, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=164, table=TestAcidGuarantees in 1.0750 sec 2024-11-24T22:16:47,521 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37701 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=164 2024-11-24T22:16:47,521 INFO [RPCClient-NioEventLoopGroup-4-12 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-24T22:16:47,522 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37701 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-24T22:16:47,523 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37701 {}] procedure2.ProcedureExecutor(1139): Stored pid=166, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=166, table=TestAcidGuarantees 2024-11-24T22:16:47,523 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37701 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=166 2024-11-24T22:16:47,523 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=166, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=166, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-24T22:16:47,524 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=166, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=166, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-24T22:16:47,524 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=167, ppid=166, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-24T22:16:47,574 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40087 {}] regionserver.HRegion(8855): Flush requested on 8ca644d2cada6dc44b363b7383dcd4fe 2024-11-24T22:16:47,574 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 8ca644d2cada6dc44b363b7383dcd4fe 3/3 column families, dataSize=114.05 KB heapSize=339.42 KB 2024-11-24T22:16:47,576 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/bd6592c37bb2:0-4 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1732486607244, max timestamp=1732486607573Num uniques 859; 2024-11-24T22:16:47,576 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/bd6592c37bb2:0-3 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1732486607244, max timestamp=1732486607573Num uniques 859; 2024-11-24T22:16:47,576 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/bd6592c37bb2:0-3 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-11-24T22:16:47,576 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/bd6592c37bb2:0-4 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-11-24T22:16:47,577 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 8ca644d2cada6dc44b363b7383dcd4fe, store=A 2024-11-24T22:16:47,577 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/bd6592c37bb2:0-0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1732486607244, max timestamp=1732486607573Num uniques 859; 2024-11-24T22:16:47,577 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/bd6592c37bb2:0-0 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-11-24T22:16:47,577 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-24T22:16:47,577 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 8ca644d2cada6dc44b363b7383dcd4fe, store=B 2024-11-24T22:16:47,577 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-24T22:16:47,577 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 8ca644d2cada6dc44b363b7383dcd4fe, store=C 2024-11-24T22:16:47,577 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-24T22:16:47,592 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024112491b71ebac22b471cac5be7cc16cbdf76_8ca644d2cada6dc44b363b7383dcd4fe is 50, key is test_row_0/A:col10/1732486607244/Put/seqid=0 2024-11-24T22:16:47,596 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45929 is added to blk_1073742474_1650 (size=12454) 2024-11-24T22:16:47,596 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T22:16:47,599 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024112491b71ebac22b471cac5be7cc16cbdf76_8ca644d2cada6dc44b363b7383dcd4fe to hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024112491b71ebac22b471cac5be7cc16cbdf76_8ca644d2cada6dc44b363b7383dcd4fe 2024-11-24T22:16:47,601 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/8ca644d2cada6dc44b363b7383dcd4fe/.tmp/A/596287ff0e7a4c05939060cb18065271, store: [table=TestAcidGuarantees family=A region=8ca644d2cada6dc44b363b7383dcd4fe] 2024-11-24T22:16:47,601 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/8ca644d2cada6dc44b363b7383dcd4fe/.tmp/A/596287ff0e7a4c05939060cb18065271 is 175, key is test_row_0/A:col10/1732486607244/Put/seqid=0 2024-11-24T22:16:47,604 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45929 is added to blk_1073742475_1651 (size=31255) 2024-11-24T22:16:47,605 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=815, memsize=11.2 K, hasBloomFilter=true, into tmp file hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/8ca644d2cada6dc44b363b7383dcd4fe/.tmp/A/596287ff0e7a4c05939060cb18065271 2024-11-24T22:16:47,611 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/8ca644d2cada6dc44b363b7383dcd4fe/.tmp/B/bfe3e61f56434ab781a4929b3389a3c6 is 50, key is test_row_0/B:col10/1732486607244/Put/seqid=0 2024-11-24T22:16:47,616 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/bd6592c37bb2:0-2 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=259, cellSize=11855, totalHeapSize=27176, min timestamp=1732486607597, max timestamp=1732486607612Num uniques 859; 2024-11-24T22:16:47,616 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/bd6592c37bb2:0-2 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=11855, suffix heap size=117064,new segment heap size=27176 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=259 2024-11-24T22:16:47,616 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/bd6592c37bb2:0-1 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=259, cellSize=11855, totalHeapSize=27176, min timestamp=1732486607597, max timestamp=1732486607612Num uniques 859; 2024-11-24T22:16:47,616 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/bd6592c37bb2:0-1 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=11855, suffix heap size=117064,new segment heap size=27176 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=259 2024-11-24T22:16:47,616 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/bd6592c37bb2:0-6 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=259, cellSize=11855, totalHeapSize=27176, min timestamp=1732486607597, max timestamp=1732486607612Num uniques 859; 2024-11-24T22:16:47,616 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/bd6592c37bb2:0-6 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=11855, suffix heap size=117064,new segment heap size=27176 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=259 2024-11-24T22:16:47,621 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45929 is added to blk_1073742476_1652 (size=12301) 2024-11-24T22:16:47,631 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37701 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=166 2024-11-24T22:16:47,641 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40087 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8ca644d2cada6dc44b363b7383dcd4fe, server=bd6592c37bb2,40087,1732486423713 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-24T22:16:47,641 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40087 {}] ipc.CallRunner(138): callId: 165 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49060 deadline: 1732486667634, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8ca644d2cada6dc44b363b7383dcd4fe, server=bd6592c37bb2,40087,1732486423713 2024-11-24T22:16:47,641 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40087 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8ca644d2cada6dc44b363b7383dcd4fe, server=bd6592c37bb2,40087,1732486423713 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-24T22:16:47,641 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40087 {}] ipc.CallRunner(138): callId: 249 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49050 deadline: 1732486667635, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8ca644d2cada6dc44b363b7383dcd4fe, server=bd6592c37bb2,40087,1732486423713 2024-11-24T22:16:47,642 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732486585339.8ca644d2cada6dc44b363b7383dcd4fe., hostname=bd6592c37bb2,40087,1732486423713, seqNum=5 , the old value is region=TestAcidGuarantees,,1732486585339.8ca644d2cada6dc44b363b7383dcd4fe., hostname=bd6592c37bb2,40087,1732486423713, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8ca644d2cada6dc44b363b7383dcd4fe, server=bd6592c37bb2,40087,1732486423713 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T22:16:47,642 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732486585339.8ca644d2cada6dc44b363b7383dcd4fe., hostname=bd6592c37bb2,40087,1732486423713, seqNum=5 , the old value is region=TestAcidGuarantees,,1732486585339.8ca644d2cada6dc44b363b7383dcd4fe., hostname=bd6592c37bb2,40087,1732486423713, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8ca644d2cada6dc44b363b7383dcd4fe, server=bd6592c37bb2,40087,1732486423713 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T22:16:47,642 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732486585339.8ca644d2cada6dc44b363b7383dcd4fe., hostname=bd6592c37bb2,40087,1732486423713, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8ca644d2cada6dc44b363b7383dcd4fe, server=bd6592c37bb2,40087,1732486423713 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T22:16:47,642 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732486585339.8ca644d2cada6dc44b363b7383dcd4fe., hostname=bd6592c37bb2,40087,1732486423713, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8ca644d2cada6dc44b363b7383dcd4fe, server=bd6592c37bb2,40087,1732486423713 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T22:16:47,642 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732486585339.8ca644d2cada6dc44b363b7383dcd4fe., hostname=bd6592c37bb2,40087,1732486423713, seqNum=5 because the exception is null or not the one we care about 2024-11-24T22:16:47,642 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732486585339.8ca644d2cada6dc44b363b7383dcd4fe., hostname=bd6592c37bb2,40087,1732486423713, seqNum=5 because the exception is null or not the one we care about 2024-11-24T22:16:47,644 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40087 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8ca644d2cada6dc44b363b7383dcd4fe, server=bd6592c37bb2,40087,1732486423713 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-24T22:16:47,644 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40087 {}] ipc.CallRunner(138): callId: 221 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48986 deadline: 1732486667637, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8ca644d2cada6dc44b363b7383dcd4fe, server=bd6592c37bb2,40087,1732486423713 2024-11-24T22:16:47,644 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40087 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8ca644d2cada6dc44b363b7383dcd4fe, server=bd6592c37bb2,40087,1732486423713 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-24T22:16:47,644 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732486585339.8ca644d2cada6dc44b363b7383dcd4fe., hostname=bd6592c37bb2,40087,1732486423713, seqNum=5 , the old value is region=TestAcidGuarantees,,1732486585339.8ca644d2cada6dc44b363b7383dcd4fe., hostname=bd6592c37bb2,40087,1732486423713, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8ca644d2cada6dc44b363b7383dcd4fe, server=bd6592c37bb2,40087,1732486423713 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T22:16:47,644 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732486585339.8ca644d2cada6dc44b363b7383dcd4fe., hostname=bd6592c37bb2,40087,1732486423713, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8ca644d2cada6dc44b363b7383dcd4fe, server=bd6592c37bb2,40087,1732486423713 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T22:16:47,644 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40087 {}] ipc.CallRunner(138): callId: 138 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49026 deadline: 1732486667638, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8ca644d2cada6dc44b363b7383dcd4fe, server=bd6592c37bb2,40087,1732486423713 2024-11-24T22:16:47,644 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732486585339.8ca644d2cada6dc44b363b7383dcd4fe., hostname=bd6592c37bb2,40087,1732486423713, seqNum=5 because the exception is null or not the one we care about 2024-11-24T22:16:47,644 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/bd6592c37bb2:0-5 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=268, cellSize=12268, totalHeapSize=28112, min timestamp=1732486607629, max timestamp=1732486607639Num uniques 1118; 2024-11-24T22:16:47,645 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732486585339.8ca644d2cada6dc44b363b7383dcd4fe., hostname=bd6592c37bb2,40087,1732486423713, seqNum=5 , the old value is region=TestAcidGuarantees,,1732486585339.8ca644d2cada6dc44b363b7383dcd4fe., hostname=bd6592c37bb2,40087,1732486423713, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8ca644d2cada6dc44b363b7383dcd4fe, server=bd6592c37bb2,40087,1732486423713 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T22:16:47,645 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/bd6592c37bb2:0-5 {}] regionserver.CompactionPipeline(163): Suffix data size=51198, new segment data size=12268, suffix heap size=144240,new segment heap size=28112 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1118, new segment cells count=268 2024-11-24T22:16:47,645 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732486585339.8ca644d2cada6dc44b363b7383dcd4fe., hostname=bd6592c37bb2,40087,1732486423713, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8ca644d2cada6dc44b363b7383dcd4fe, server=bd6592c37bb2,40087,1732486423713 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T22:16:47,645 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/bd6592c37bb2:0-7 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=268, cellSize=12268, totalHeapSize=28112, min timestamp=1732486607629, max timestamp=1732486607639Num uniques 1118; 2024-11-24T22:16:47,645 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732486585339.8ca644d2cada6dc44b363b7383dcd4fe., hostname=bd6592c37bb2,40087,1732486423713, seqNum=5 because the exception is null or not the one we care about 2024-11-24T22:16:47,645 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/bd6592c37bb2:0-7 {}] regionserver.CompactionPipeline(163): Suffix data size=51198, new segment data size=12268, suffix heap size=144240,new segment heap size=28112 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1118, new segment cells count=268 2024-11-24T22:16:47,645 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/bd6592c37bb2:0-9 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=268, cellSize=12268, totalHeapSize=28112, min timestamp=1732486607629, max timestamp=1732486607639Num uniques 1118; 2024-11-24T22:16:47,645 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/bd6592c37bb2:0-9 {}] regionserver.CompactionPipeline(163): Suffix data size=51198, new segment data size=12268, suffix heap size=144240,new segment heap size=28112 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1118, new segment cells count=268 2024-11-24T22:16:47,676 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=40087 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=167 2024-11-24T22:16:47,676 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bd6592c37bb2:0-2 {event_type=RS_FLUSH_REGIONS, pid=167}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732486585339.8ca644d2cada6dc44b363b7383dcd4fe. 2024-11-24T22:16:47,676 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bd6592c37bb2:0-2 {event_type=RS_FLUSH_REGIONS, pid=167}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732486585339.8ca644d2cada6dc44b363b7383dcd4fe. as already flushing 2024-11-24T22:16:47,676 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bd6592c37bb2:0-2 {event_type=RS_FLUSH_REGIONS, pid=167}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732486585339.8ca644d2cada6dc44b363b7383dcd4fe. 2024-11-24T22:16:47,676 ERROR [RS_FLUSH_OPERATIONS-regionserver/bd6592c37bb2:0-2 {event_type=RS_FLUSH_REGIONS, pid=167}] handler.RSProcedureHandler(58): pid=167 java.io.IOException: Unable to complete flush {ENCODED => 8ca644d2cada6dc44b363b7383dcd4fe, NAME => 'TestAcidGuarantees,,1732486585339.8ca644d2cada6dc44b363b7383dcd4fe.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T22:16:47,677 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bd6592c37bb2:0-2 {event_type=RS_FLUSH_REGIONS, pid=167}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=167 java.io.IOException: Unable to complete flush {ENCODED => 8ca644d2cada6dc44b363b7383dcd4fe, NAME => 'TestAcidGuarantees,,1732486585339.8ca644d2cada6dc44b363b7383dcd4fe.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T22:16:47,677 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37701 {}] master.HMaster(4177): Remote procedure failed, pid=167 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 8ca644d2cada6dc44b363b7383dcd4fe, NAME => 'TestAcidGuarantees,,1732486585339.8ca644d2cada6dc44b363b7383dcd4fe.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 8ca644d2cada6dc44b363b7383dcd4fe, NAME => 'TestAcidGuarantees,,1732486585339.8ca644d2cada6dc44b363b7383dcd4fe.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T22:16:47,746 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/bd6592c37bb2:0-3 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=177, cellSize=8102, totalHeapSize=18648, min timestamp=1732486607722, max timestamp=1732486607740Num uniques 1127; 2024-11-24T22:16:47,746 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/bd6592c37bb2:0-8 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=177, cellSize=8102, totalHeapSize=18648, min timestamp=1732486607722, max timestamp=1732486607740Num uniques 1127; 2024-11-24T22:16:47,746 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/bd6592c37bb2:0-3 {}] regionserver.CompactionPipeline(163): Suffix data size=51612, new segment data size=8102, suffix heap size=145176,new segment heap size=18648 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1127, new segment cells count=177 2024-11-24T22:16:47,746 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/bd6592c37bb2:0-8 {}] regionserver.CompactionPipeline(163): Suffix data size=51612, new segment data size=8102, suffix heap size=145176,new segment heap size=18648 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1127, new segment cells count=177 2024-11-24T22:16:47,746 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/bd6592c37bb2:0-4 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=177, cellSize=8102, totalHeapSize=18648, min timestamp=1732486607722, max timestamp=1732486607740Num uniques 1127; 2024-11-24T22:16:47,746 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/bd6592c37bb2:0-4 {}] regionserver.CompactionPipeline(163): Suffix data size=51612, new segment data size=8102, suffix heap size=145176,new segment heap size=18648 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1127, new segment cells count=177 2024-11-24T22:16:47,782 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40087 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8ca644d2cada6dc44b363b7383dcd4fe, server=bd6592c37bb2,40087,1732486423713 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-24T22:16:47,783 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40087 {}] ipc.CallRunner(138): callId: 225 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48986 deadline: 1732486667773, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8ca644d2cada6dc44b363b7383dcd4fe, server=bd6592c37bb2,40087,1732486423713 2024-11-24T22:16:47,783 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40087 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8ca644d2cada6dc44b363b7383dcd4fe, server=bd6592c37bb2,40087,1732486423713 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-24T22:16:47,783 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732486585339.8ca644d2cada6dc44b363b7383dcd4fe., hostname=bd6592c37bb2,40087,1732486423713, seqNum=5 , the old value is region=TestAcidGuarantees,,1732486585339.8ca644d2cada6dc44b363b7383dcd4fe., hostname=bd6592c37bb2,40087,1732486423713, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8ca644d2cada6dc44b363b7383dcd4fe, server=bd6592c37bb2,40087,1732486423713 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T22:16:47,783 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732486585339.8ca644d2cada6dc44b363b7383dcd4fe., hostname=bd6592c37bb2,40087,1732486423713, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8ca644d2cada6dc44b363b7383dcd4fe, server=bd6592c37bb2,40087,1732486423713 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T22:16:47,783 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40087 {}] ipc.CallRunner(138): callId: 142 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49026 deadline: 1732486667774, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8ca644d2cada6dc44b363b7383dcd4fe, server=bd6592c37bb2,40087,1732486423713 2024-11-24T22:16:47,783 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732486585339.8ca644d2cada6dc44b363b7383dcd4fe., hostname=bd6592c37bb2,40087,1732486423713, seqNum=5 because the exception is null or not the one we care about 2024-11-24T22:16:47,783 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732486585339.8ca644d2cada6dc44b363b7383dcd4fe., hostname=bd6592c37bb2,40087,1732486423713, seqNum=5 , the old value is region=TestAcidGuarantees,,1732486585339.8ca644d2cada6dc44b363b7383dcd4fe., hostname=bd6592c37bb2,40087,1732486423713, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8ca644d2cada6dc44b363b7383dcd4fe, server=bd6592c37bb2,40087,1732486423713 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T22:16:47,783 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732486585339.8ca644d2cada6dc44b363b7383dcd4fe., hostname=bd6592c37bb2,40087,1732486423713, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8ca644d2cada6dc44b363b7383dcd4fe, server=bd6592c37bb2,40087,1732486423713 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T22:16:47,783 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732486585339.8ca644d2cada6dc44b363b7383dcd4fe., hostname=bd6592c37bb2,40087,1732486423713, seqNum=5 because the exception is null or not the one we care about 2024-11-24T22:16:47,783 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/bd6592c37bb2:0-0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=186, cellSize=8516, totalHeapSize=19584, min timestamp=1732486607769, max timestamp=1732486607778Num uniques 1036; 2024-11-24T22:16:47,783 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/bd6592c37bb2:0-0 {}] regionserver.CompactionPipeline(163): Suffix data size=47446, new segment data size=8516, suffix heap size=135712,new segment heap size=19584 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1036, new segment cells count=186 2024-11-24T22:16:47,783 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/bd6592c37bb2:0-2 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=186, cellSize=8516, totalHeapSize=19584, min timestamp=1732486607769, max timestamp=1732486607778Num uniques 1036; 2024-11-24T22:16:47,783 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/bd6592c37bb2:0-2 {}] regionserver.CompactionPipeline(163): Suffix data size=47446, new segment data size=8516, suffix heap size=135712,new segment heap size=19584 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1036, new segment cells count=186 2024-11-24T22:16:47,784 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/bd6592c37bb2:0-1 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=186, cellSize=8516, totalHeapSize=19584, min timestamp=1732486607769, max timestamp=1732486607778Num uniques 1036; 2024-11-24T22:16:47,784 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/bd6592c37bb2:0-1 {}] regionserver.CompactionPipeline(163): Suffix data size=47446, new segment data size=8516, suffix heap size=135712,new segment heap size=19584 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1036, new segment cells count=186 2024-11-24T22:16:47,821 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/bd6592c37bb2:0-5 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=195, cellSize=8930, totalHeapSize=20520, min timestamp=1732486607799, max timestamp=1732486607817Num uniques 1045; 2024-11-24T22:16:47,821 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/bd6592c37bb2:0-5 {}] regionserver.CompactionPipeline(163): Suffix data size=47860, new segment data size=8930, suffix heap size=136648,new segment heap size=20520 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1045, new segment cells count=195 2024-11-24T22:16:47,821 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/bd6592c37bb2:0-6 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=195, cellSize=8930, totalHeapSize=20520, min timestamp=1732486607799, max timestamp=1732486607817Num uniques 1045; 2024-11-24T22:16:47,821 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/bd6592c37bb2:0-6 {}] regionserver.CompactionPipeline(163): Suffix data size=47860, new segment data size=8930, suffix heap size=136648,new segment heap size=20520 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1045, new segment cells count=195 2024-11-24T22:16:47,821 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/bd6592c37bb2:0-7 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=195, cellSize=8930, totalHeapSize=20520, min timestamp=1732486607799, max timestamp=1732486607817Num uniques 1045; 2024-11-24T22:16:47,822 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/bd6592c37bb2:0-7 {}] regionserver.CompactionPipeline(163): Suffix data size=47860, new segment data size=8930, suffix heap size=136648,new segment heap size=20520 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1045, new segment cells count=195 2024-11-24T22:16:47,829 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=40087 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=167 2024-11-24T22:16:47,829 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bd6592c37bb2:0-0 {event_type=RS_FLUSH_REGIONS, pid=167}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732486585339.8ca644d2cada6dc44b363b7383dcd4fe. 2024-11-24T22:16:47,829 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bd6592c37bb2:0-0 {event_type=RS_FLUSH_REGIONS, pid=167}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732486585339.8ca644d2cada6dc44b363b7383dcd4fe. as already flushing 2024-11-24T22:16:47,829 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bd6592c37bb2:0-0 {event_type=RS_FLUSH_REGIONS, pid=167}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732486585339.8ca644d2cada6dc44b363b7383dcd4fe. 2024-11-24T22:16:47,829 ERROR [RS_FLUSH_OPERATIONS-regionserver/bd6592c37bb2:0-0 {event_type=RS_FLUSH_REGIONS, pid=167}] handler.RSProcedureHandler(58): pid=167 java.io.IOException: Unable to complete flush {ENCODED => 8ca644d2cada6dc44b363b7383dcd4fe, NAME => 'TestAcidGuarantees,,1732486585339.8ca644d2cada6dc44b363b7383dcd4fe.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T22:16:47,829 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bd6592c37bb2:0-0 {event_type=RS_FLUSH_REGIONS, pid=167}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=167 java.io.IOException: Unable to complete flush {ENCODED => 8ca644d2cada6dc44b363b7383dcd4fe, NAME => 'TestAcidGuarantees,,1732486585339.8ca644d2cada6dc44b363b7383dcd4fe.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T22:16:47,830 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37701 {}] master.HMaster(4177): Remote procedure failed, pid=167 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 8ca644d2cada6dc44b363b7383dcd4fe, NAME => 'TestAcidGuarantees,,1732486585339.8ca644d2cada6dc44b363b7383dcd4fe.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 8ca644d2cada6dc44b363b7383dcd4fe, NAME => 'TestAcidGuarantees,,1732486585339.8ca644d2cada6dc44b363b7383dcd4fe.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T22:16:47,841 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37701 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=166 2024-11-24T22:16:47,863 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/bd6592c37bb2:0-9 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=204, cellSize=9340, totalHeapSize=21456, min timestamp=1732486607836, max timestamp=1732486607859Num uniques 1054; 2024-11-24T22:16:47,863 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/bd6592c37bb2:0-9 {}] regionserver.CompactionPipeline(163): Suffix data size=48270, new segment data size=9340, suffix heap size=137584,new segment heap size=21456 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1054, new segment cells count=204 2024-11-24T22:16:47,864 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/bd6592c37bb2:0-3 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=204, cellSize=9340, totalHeapSize=21456, min timestamp=1732486607836, max timestamp=1732486607859Num uniques 1054; 2024-11-24T22:16:47,864 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/bd6592c37bb2:0-3 {}] regionserver.CompactionPipeline(163): Suffix data size=48270, new segment data size=9340, suffix heap size=137584,new segment heap size=21456 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1054, new segment cells count=204 2024-11-24T22:16:47,864 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/bd6592c37bb2:0-8 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=204, cellSize=9340, totalHeapSize=21456, min timestamp=1732486607836, max timestamp=1732486607859Num uniques 1054; 2024-11-24T22:16:47,864 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/bd6592c37bb2:0-8 {}] regionserver.CompactionPipeline(163): Suffix data size=48270, new segment data size=9340, suffix heap size=137584,new segment heap size=21456 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1054, new segment cells count=204 2024-11-24T22:16:47,904 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/bd6592c37bb2:0-4 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=313, cellSize=14328, totalHeapSize=32792, min timestamp=1732486607885, max timestamp=1732486607899Num uniques 1063; 2024-11-24T22:16:47,904 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/bd6592c37bb2:0-4 {}] regionserver.CompactionPipeline(163): Suffix data size=48678, new segment data size=14328, suffix heap size=138520,new segment heap size=32792 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1063, new segment cells count=313 2024-11-24T22:16:47,904 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/bd6592c37bb2:0-0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=313, cellSize=14328, totalHeapSize=32792, min timestamp=1732486607885, max timestamp=1732486607899Num uniques 1063; 2024-11-24T22:16:47,904 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/bd6592c37bb2:0-0 {}] regionserver.CompactionPipeline(163): Suffix data size=48678, new segment data size=14328, suffix heap size=138520,new segment heap size=32792 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1063, new segment cells count=313 2024-11-24T22:16:47,906 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/bd6592c37bb2:0-2 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=313, cellSize=14328, totalHeapSize=32792, min timestamp=1732486607885, max timestamp=1732486607899Num uniques 1063; 2024-11-24T22:16:47,906 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/bd6592c37bb2:0-2 {}] regionserver.CompactionPipeline(163): Suffix data size=48678, new segment data size=14328, suffix heap size=138520,new segment heap size=32792 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1063, new segment cells count=313 2024-11-24T22:16:47,934 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40087 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8ca644d2cada6dc44b363b7383dcd4fe, server=bd6592c37bb2,40087,1732486423713 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-24T22:16:47,934 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40087 {}] ipc.CallRunner(138): callId: 230 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48986 deadline: 1732486667927, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8ca644d2cada6dc44b363b7383dcd4fe, server=bd6592c37bb2,40087,1732486423713 2024-11-24T22:16:47,935 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40087 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8ca644d2cada6dc44b363b7383dcd4fe, server=bd6592c37bb2,40087,1732486423713 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-24T22:16:47,935 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732486585339.8ca644d2cada6dc44b363b7383dcd4fe., hostname=bd6592c37bb2,40087,1732486423713, seqNum=5 , the old value is region=TestAcidGuarantees,,1732486585339.8ca644d2cada6dc44b363b7383dcd4fe., hostname=bd6592c37bb2,40087,1732486423713, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8ca644d2cada6dc44b363b7383dcd4fe, server=bd6592c37bb2,40087,1732486423713 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T22:16:47,935 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732486585339.8ca644d2cada6dc44b363b7383dcd4fe., hostname=bd6592c37bb2,40087,1732486423713, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8ca644d2cada6dc44b363b7383dcd4fe, server=bd6592c37bb2,40087,1732486423713 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T22:16:47,935 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40087 {}] ipc.CallRunner(138): callId: 273 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49050 deadline: 1732486667928, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8ca644d2cada6dc44b363b7383dcd4fe, server=bd6592c37bb2,40087,1732486423713 2024-11-24T22:16:47,935 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732486585339.8ca644d2cada6dc44b363b7383dcd4fe., hostname=bd6592c37bb2,40087,1732486423713, seqNum=5 because the exception is null or not the one we care about 2024-11-24T22:16:47,936 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40087 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8ca644d2cada6dc44b363b7383dcd4fe, server=bd6592c37bb2,40087,1732486423713 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-24T22:16:47,936 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732486585339.8ca644d2cada6dc44b363b7383dcd4fe., hostname=bd6592c37bb2,40087,1732486423713, seqNum=5 , the old value is region=TestAcidGuarantees,,1732486585339.8ca644d2cada6dc44b363b7383dcd4fe., hostname=bd6592c37bb2,40087,1732486423713, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8ca644d2cada6dc44b363b7383dcd4fe, server=bd6592c37bb2,40087,1732486423713 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T22:16:47,936 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732486585339.8ca644d2cada6dc44b363b7383dcd4fe., hostname=bd6592c37bb2,40087,1732486423713, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8ca644d2cada6dc44b363b7383dcd4fe, server=bd6592c37bb2,40087,1732486423713 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T22:16:47,936 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40087 {}] ipc.CallRunner(138): callId: 147 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49026 deadline: 1732486667929, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8ca644d2cada6dc44b363b7383dcd4fe, server=bd6592c37bb2,40087,1732486423713 2024-11-24T22:16:47,936 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732486585339.8ca644d2cada6dc44b363b7383dcd4fe., hostname=bd6592c37bb2,40087,1732486423713, seqNum=5 because the exception is null or not the one we care about 2024-11-24T22:16:47,936 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732486585339.8ca644d2cada6dc44b363b7383dcd4fe., hostname=bd6592c37bb2,40087,1732486423713, seqNum=5 , the old value is region=TestAcidGuarantees,,1732486585339.8ca644d2cada6dc44b363b7383dcd4fe., hostname=bd6592c37bb2,40087,1732486423713, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8ca644d2cada6dc44b363b7383dcd4fe, server=bd6592c37bb2,40087,1732486423713 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T22:16:47,936 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732486585339.8ca644d2cada6dc44b363b7383dcd4fe., hostname=bd6592c37bb2,40087,1732486423713, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8ca644d2cada6dc44b363b7383dcd4fe, server=bd6592c37bb2,40087,1732486423713 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T22:16:47,936 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732486585339.8ca644d2cada6dc44b363b7383dcd4fe., hostname=bd6592c37bb2,40087,1732486423713, seqNum=5 because the exception is null or not the one we care about 2024-11-24T22:16:47,938 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/bd6592c37bb2:0-5 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=222, cellSize=10162, totalHeapSize=23328, min timestamp=1732486607924, max timestamp=1732486607933Num uniques 1172; 2024-11-24T22:16:47,938 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/bd6592c37bb2:0-1 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=222, cellSize=10162, totalHeapSize=23328, min timestamp=1732486607924, max timestamp=1732486607933Num uniques 1172; 2024-11-24T22:16:47,938 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/bd6592c37bb2:0-5 {}] regionserver.CompactionPipeline(163): Suffix data size=53672, new segment data size=10162, suffix heap size=149856,new segment heap size=23328 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1172, new segment cells count=222 2024-11-24T22:16:47,938 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/bd6592c37bb2:0-1 {}] regionserver.CompactionPipeline(163): Suffix data size=53672, new segment data size=10162, suffix heap size=149856,new segment heap size=23328 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1172, new segment cells count=222 2024-11-24T22:16:47,938 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/bd6592c37bb2:0-6 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=222, cellSize=10162, totalHeapSize=23328, min timestamp=1732486607924, max timestamp=1732486607933Num uniques 1172; 2024-11-24T22:16:47,938 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/bd6592c37bb2:0-6 {}] regionserver.CompactionPipeline(163): Suffix data size=53672, new segment data size=10162, suffix heap size=149856,new segment heap size=23328 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1172, new segment cells count=222 2024-11-24T22:16:47,982 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=40087 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=167 2024-11-24T22:16:47,982 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bd6592c37bb2:0-1 {event_type=RS_FLUSH_REGIONS, pid=167}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732486585339.8ca644d2cada6dc44b363b7383dcd4fe. 2024-11-24T22:16:47,982 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bd6592c37bb2:0-1 {event_type=RS_FLUSH_REGIONS, pid=167}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732486585339.8ca644d2cada6dc44b363b7383dcd4fe. as already flushing 2024-11-24T22:16:47,982 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bd6592c37bb2:0-1 {event_type=RS_FLUSH_REGIONS, pid=167}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732486585339.8ca644d2cada6dc44b363b7383dcd4fe. 2024-11-24T22:16:47,982 ERROR [RS_FLUSH_OPERATIONS-regionserver/bd6592c37bb2:0-1 {event_type=RS_FLUSH_REGIONS, pid=167}] handler.RSProcedureHandler(58): pid=167 java.io.IOException: Unable to complete flush {ENCODED => 8ca644d2cada6dc44b363b7383dcd4fe, NAME => 'TestAcidGuarantees,,1732486585339.8ca644d2cada6dc44b363b7383dcd4fe.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T22:16:47,982 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bd6592c37bb2:0-1 {event_type=RS_FLUSH_REGIONS, pid=167}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=167 java.io.IOException: Unable to complete flush {ENCODED => 8ca644d2cada6dc44b363b7383dcd4fe, NAME => 'TestAcidGuarantees,,1732486585339.8ca644d2cada6dc44b363b7383dcd4fe.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T22:16:47,983 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37701 {}] master.HMaster(4177): Remote procedure failed, pid=167 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 8ca644d2cada6dc44b363b7383dcd4fe, NAME => 'TestAcidGuarantees,,1732486585339.8ca644d2cada6dc44b363b7383dcd4fe.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 8ca644d2cada6dc44b363b7383dcd4fe, NAME => 'TestAcidGuarantees,,1732486585339.8ca644d2cada6dc44b363b7383dcd4fe.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T22:16:47,996 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/bd6592c37bb2:0-9 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=231, cellSize=10576, totalHeapSize=24264, min timestamp=1732486607965, max timestamp=1732486607992Num uniques 1081; 2024-11-24T22:16:47,996 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/bd6592c37bb2:0-9 {}] regionserver.CompactionPipeline(163): Suffix data size=49506, new segment data size=10576, suffix heap size=140392,new segment heap size=24264 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1081, new segment cells count=231 2024-11-24T22:16:47,997 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/bd6592c37bb2:0-7 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=231, cellSize=10576, totalHeapSize=24264, min timestamp=1732486607965, max timestamp=1732486607992Num uniques 1081; 2024-11-24T22:16:47,997 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/bd6592c37bb2:0-7 {}] regionserver.CompactionPipeline(163): Suffix data size=49506, new segment data size=10576, suffix heap size=140392,new segment heap size=24264 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1081, new segment cells count=231 2024-11-24T22:16:47,997 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/bd6592c37bb2:0-3 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=231, cellSize=10576, totalHeapSize=24264, min timestamp=1732486607965, max timestamp=1732486607992Num uniques 1081; 2024-11-24T22:16:47,997 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/bd6592c37bb2:0-3 {}] regionserver.CompactionPipeline(163): Suffix data size=49506, new segment data size=10576, suffix heap size=140392,new segment heap size=24264 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1081, new segment cells count=231 2024-11-24T22:16:48,022 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=11.18 KB at sequenceid=815 (bloomFilter=true), to=hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/8ca644d2cada6dc44b363b7383dcd4fe/.tmp/B/bfe3e61f56434ab781a4929b3389a3c6 2024-11-24T22:16:48,028 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/8ca644d2cada6dc44b363b7383dcd4fe/.tmp/C/40a953156e07441eb8fc6867ae9b1954 is 50, key is test_row_0/C:col10/1732486607244/Put/seqid=0 2024-11-24T22:16:48,048 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/bd6592c37bb2:0-8 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=240, cellSize=10990, totalHeapSize=25200, min timestamp=1732486608015, max timestamp=1732486608043Num uniques 1090; 2024-11-24T22:16:48,048 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/bd6592c37bb2:0-8 {}] regionserver.CompactionPipeline(163): Suffix data size=49920, new segment data size=10990, suffix heap size=141328,new segment heap size=25200 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1090, new segment cells count=240 2024-11-24T22:16:48,048 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/bd6592c37bb2:0-4 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=240, cellSize=10990, totalHeapSize=25200, min timestamp=1732486608015, max timestamp=1732486608043Num uniques 1090; 2024-11-24T22:16:48,048 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/bd6592c37bb2:0-4 {}] regionserver.CompactionPipeline(163): Suffix data size=49920, new segment data size=10990, suffix heap size=141328,new segment heap size=25200 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1090, new segment cells count=240 2024-11-24T22:16:48,048 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/bd6592c37bb2:0-0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=240, cellSize=10990, totalHeapSize=25200, min timestamp=1732486608015, max timestamp=1732486608043Num uniques 1090; 2024-11-24T22:16:48,048 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/bd6592c37bb2:0-0 {}] regionserver.CompactionPipeline(163): Suffix data size=49920, new segment data size=10990, suffix heap size=141328,new segment heap size=25200 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1090, new segment cells count=240 2024-11-24T22:16:48,054 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45929 is added to blk_1073742477_1653 (size=12301) 2024-11-24T22:16:48,054 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=11.18 KB at sequenceid=815 (bloomFilter=true), to=hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/8ca644d2cada6dc44b363b7383dcd4fe/.tmp/C/40a953156e07441eb8fc6867ae9b1954 2024-11-24T22:16:48,058 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/8ca644d2cada6dc44b363b7383dcd4fe/.tmp/A/596287ff0e7a4c05939060cb18065271 as hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/8ca644d2cada6dc44b363b7383dcd4fe/A/596287ff0e7a4c05939060cb18065271 2024-11-24T22:16:48,063 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/8ca644d2cada6dc44b363b7383dcd4fe/A/596287ff0e7a4c05939060cb18065271, entries=150, sequenceid=815, filesize=30.5 K 2024-11-24T22:16:48,064 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/8ca644d2cada6dc44b363b7383dcd4fe/.tmp/B/bfe3e61f56434ab781a4929b3389a3c6 as hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/8ca644d2cada6dc44b363b7383dcd4fe/B/bfe3e61f56434ab781a4929b3389a3c6 2024-11-24T22:16:48,067 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/8ca644d2cada6dc44b363b7383dcd4fe/B/bfe3e61f56434ab781a4929b3389a3c6, entries=150, sequenceid=815, filesize=12.0 K 2024-11-24T22:16:48,068 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/8ca644d2cada6dc44b363b7383dcd4fe/.tmp/C/40a953156e07441eb8fc6867ae9b1954 as hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/8ca644d2cada6dc44b363b7383dcd4fe/C/40a953156e07441eb8fc6867ae9b1954 2024-11-24T22:16:48,071 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/8ca644d2cada6dc44b363b7383dcd4fe/C/40a953156e07441eb8fc6867ae9b1954, entries=150, sequenceid=815, filesize=12.0 K 2024-11-24T22:16:48,072 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~33.54 KB/34350, heapSize ~86.11 KB/88176, currentSize=114.05 KB/116790 for 8ca644d2cada6dc44b363b7383dcd4fe in 498ms, sequenceid=815, compaction requested=true 2024-11-24T22:16:48,072 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 8ca644d2cada6dc44b363b7383dcd4fe: 2024-11-24T22:16:48,072 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40087 {}] regionserver.HRegion(8855): Flush requested on 8ca644d2cada6dc44b363b7383dcd4fe 2024-11-24T22:16:48,072 DEBUG [RS:0;bd6592c37bb2:40087-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-24T22:16:48,073 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 8ca644d2cada6dc44b363b7383dcd4fe:A, priority=-2147483648, current under compaction store size is 1 2024-11-24T22:16:48,073 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-24T22:16:48,073 DEBUG [RS:0;bd6592c37bb2:40087-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-24T22:16:48,073 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 8ca644d2cada6dc44b363b7383dcd4fe:B, priority=-2147483648, current under compaction store size is 2 2024-11-24T22:16:48,073 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-24T22:16:48,073 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 8ca644d2cada6dc44b363b7383dcd4fe:C, priority=-2147483648, current under compaction store size is 3 2024-11-24T22:16:48,073 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-24T22:16:48,074 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 8ca644d2cada6dc44b363b7383dcd4fe 3/3 column families, dataSize=127.47 KB heapSize=357.47 KB 2024-11-24T22:16:48,074 DEBUG [RS:0;bd6592c37bb2:40087-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 134362 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-24T22:16:48,074 DEBUG [RS:0;bd6592c37bb2:40087-shortCompactions-0 {}] regionserver.HStore(1541): 8ca644d2cada6dc44b363b7383dcd4fe/A is initiating minor compaction (all files) 2024-11-24T22:16:48,074 INFO [RS:0;bd6592c37bb2:40087-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 8ca644d2cada6dc44b363b7383dcd4fe/A in TestAcidGuarantees,,1732486585339.8ca644d2cada6dc44b363b7383dcd4fe. 2024-11-24T22:16:48,074 INFO [RS:0;bd6592c37bb2:40087-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/8ca644d2cada6dc44b363b7383dcd4fe/A/aca73ecfca3c475c865c0a6c969e035c, hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/8ca644d2cada6dc44b363b7383dcd4fe/A/5b75c2f936b144c4a5e4bf896c7a3178, hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/8ca644d2cada6dc44b363b7383dcd4fe/A/edcdc9c2d1544300b712df3846855e54, hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/8ca644d2cada6dc44b363b7383dcd4fe/A/596287ff0e7a4c05939060cb18065271] into tmpdir=hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/8ca644d2cada6dc44b363b7383dcd4fe/.tmp, totalSize=131.2 K 2024-11-24T22:16:48,074 INFO [RS:0;bd6592c37bb2:40087-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=12 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1732486585339.8ca644d2cada6dc44b363b7383dcd4fe. 2024-11-24T22:16:48,074 DEBUG [RS:0;bd6592c37bb2:40087-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1732486585339.8ca644d2cada6dc44b363b7383dcd4fe. files: [hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/8ca644d2cada6dc44b363b7383dcd4fe/A/aca73ecfca3c475c865c0a6c969e035c, hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/8ca644d2cada6dc44b363b7383dcd4fe/A/5b75c2f936b144c4a5e4bf896c7a3178, hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/8ca644d2cada6dc44b363b7383dcd4fe/A/edcdc9c2d1544300b712df3846855e54, hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/8ca644d2cada6dc44b363b7383dcd4fe/A/596287ff0e7a4c05939060cb18065271] 2024-11-24T22:16:48,074 DEBUG [RS:0;bd6592c37bb2:40087-shortCompactions-0 {}] compactions.Compactor(225): Compacting aca73ecfca3c475c865c0a6c969e035c, keycount=150, bloomtype=ROW, size=31.2 K, encoding=NONE, compression=NONE, seqNum=744, earliestPutTs=1732486604909 2024-11-24T22:16:48,074 DEBUG [RS:0;bd6592c37bb2:40087-shortCompactions-0 {}] compactions.Compactor(225): Compacting 5b75c2f936b144c4a5e4bf896c7a3178, keycount=200, bloomtype=ROW, size=39.0 K, encoding=NONE, compression=NONE, seqNum=777, earliestPutTs=1732486606021 2024-11-24T22:16:48,075 DEBUG [RS:0;bd6592c37bb2:40087-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 49852 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-24T22:16:48,075 DEBUG [RS:0;bd6592c37bb2:40087-shortCompactions-0 {}] compactions.Compactor(225): Compacting edcdc9c2d1544300b712df3846855e54, keycount=150, bloomtype=ROW, size=30.5 K, encoding=NONE, compression=NONE, seqNum=793, earliestPutTs=1732486606060 2024-11-24T22:16:48,075 DEBUG [RS:0;bd6592c37bb2:40087-longCompactions-0 {}] regionserver.HStore(1541): 8ca644d2cada6dc44b363b7383dcd4fe/B is initiating minor compaction (all files) 2024-11-24T22:16:48,075 INFO [RS:0;bd6592c37bb2:40087-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 8ca644d2cada6dc44b363b7383dcd4fe/B in TestAcidGuarantees,,1732486585339.8ca644d2cada6dc44b363b7383dcd4fe. 2024-11-24T22:16:48,075 INFO [RS:0;bd6592c37bb2:40087-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/8ca644d2cada6dc44b363b7383dcd4fe/B/3b068b24fdb74c6190094f2727449194, hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/8ca644d2cada6dc44b363b7383dcd4fe/B/60a721145bed4ac1ac6c4ee15134ccdb, hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/8ca644d2cada6dc44b363b7383dcd4fe/B/8b5d192284234dd797b73ec2a04209f8, hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/8ca644d2cada6dc44b363b7383dcd4fe/B/bfe3e61f56434ab781a4929b3389a3c6] into tmpdir=hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/8ca644d2cada6dc44b363b7383dcd4fe/.tmp, totalSize=48.7 K 2024-11-24T22:16:48,075 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 8ca644d2cada6dc44b363b7383dcd4fe, store=A 2024-11-24T22:16:48,075 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-24T22:16:48,075 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 8ca644d2cada6dc44b363b7383dcd4fe, store=B 2024-11-24T22:16:48,075 DEBUG [RS:0;bd6592c37bb2:40087-shortCompactions-0 {}] compactions.Compactor(225): Compacting 596287ff0e7a4c05939060cb18065271, keycount=150, bloomtype=ROW, size=30.5 K, encoding=NONE, compression=NONE, seqNum=815, earliestPutTs=1732486607244 2024-11-24T22:16:48,075 DEBUG [RS:0;bd6592c37bb2:40087-longCompactions-0 {}] compactions.Compactor(225): Compacting 3b068b24fdb74c6190094f2727449194, keycount=150, bloomtype=ROW, size=12.6 K, encoding=NONE, compression=NONE, seqNum=744, earliestPutTs=1732486604909 2024-11-24T22:16:48,075 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-24T22:16:48,075 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 8ca644d2cada6dc44b363b7383dcd4fe, store=C 2024-11-24T22:16:48,075 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-24T22:16:48,075 DEBUG [RS:0;bd6592c37bb2:40087-longCompactions-0 {}] compactions.Compactor(225): Compacting 60a721145bed4ac1ac6c4ee15134ccdb, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=777, earliestPutTs=1732486606021 2024-11-24T22:16:48,076 DEBUG [RS:0;bd6592c37bb2:40087-longCompactions-0 {}] compactions.Compactor(225): Compacting 8b5d192284234dd797b73ec2a04209f8, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=793, earliestPutTs=1732486606060 2024-11-24T22:16:48,076 DEBUG [RS:0;bd6592c37bb2:40087-longCompactions-0 {}] compactions.Compactor(225): Compacting bfe3e61f56434ab781a4929b3389a3c6, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=815, earliestPutTs=1732486607244 2024-11-24T22:16:48,083 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241124bda66f2b082942a998944897f1893b7f_8ca644d2cada6dc44b363b7383dcd4fe is 50, key is test_row_0/A:col10/1732486608072/Put/seqid=0 2024-11-24T22:16:48,084 INFO [RS:0;bd6592c37bb2:40087-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=8ca644d2cada6dc44b363b7383dcd4fe] 2024-11-24T22:16:48,086 DEBUG [RS:0;bd6592c37bb2:40087-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e2024112406c55f645fff43e08121100b3f47612b_8ca644d2cada6dc44b363b7383dcd4fe store=[table=TestAcidGuarantees family=A region=8ca644d2cada6dc44b363b7383dcd4fe] 2024-11-24T22:16:48,087 INFO [RS:0;bd6592c37bb2:40087-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 8ca644d2cada6dc44b363b7383dcd4fe#B#compaction#550 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-24T22:16:48,088 DEBUG [RS:0;bd6592c37bb2:40087-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e2024112406c55f645fff43e08121100b3f47612b_8ca644d2cada6dc44b363b7383dcd4fe, store=[table=TestAcidGuarantees family=A region=8ca644d2cada6dc44b363b7383dcd4fe] 2024-11-24T22:16:48,088 DEBUG [RS:0;bd6592c37bb2:40087-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/8ca644d2cada6dc44b363b7383dcd4fe/.tmp/B/ed0b458c5c334334a162cfbf3d0063e1 is 50, key is test_row_0/B:col10/1732486607244/Put/seqid=0 2024-11-24T22:16:48,088 DEBUG [RS:0;bd6592c37bb2:40087-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024112406c55f645fff43e08121100b3f47612b_8ca644d2cada6dc44b363b7383dcd4fe because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=8ca644d2cada6dc44b363b7383dcd4fe] 2024-11-24T22:16:48,090 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45929 is added to blk_1073742478_1654 (size=14994) 2024-11-24T22:16:48,091 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40087 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8ca644d2cada6dc44b363b7383dcd4fe, server=bd6592c37bb2,40087,1732486423713 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-24T22:16:48,091 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40087 {}] ipc.CallRunner(138): callId: 275 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49002 deadline: 1732486668083, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8ca644d2cada6dc44b363b7383dcd4fe, server=bd6592c37bb2,40087,1732486423713 2024-11-24T22:16:48,091 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40087 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8ca644d2cada6dc44b363b7383dcd4fe, server=bd6592c37bb2,40087,1732486423713 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-24T22:16:48,091 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40087 {}] ipc.CallRunner(138): callId: 278 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49050 deadline: 1732486668084, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8ca644d2cada6dc44b363b7383dcd4fe, server=bd6592c37bb2,40087,1732486423713 2024-11-24T22:16:48,091 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732486585339.8ca644d2cada6dc44b363b7383dcd4fe., hostname=bd6592c37bb2,40087,1732486423713, seqNum=5 , the old value is region=TestAcidGuarantees,,1732486585339.8ca644d2cada6dc44b363b7383dcd4fe., hostname=bd6592c37bb2,40087,1732486423713, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8ca644d2cada6dc44b363b7383dcd4fe, server=bd6592c37bb2,40087,1732486423713 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T22:16:48,091 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732486585339.8ca644d2cada6dc44b363b7383dcd4fe., hostname=bd6592c37bb2,40087,1732486423713, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8ca644d2cada6dc44b363b7383dcd4fe, server=bd6592c37bb2,40087,1732486423713 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T22:16:48,091 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732486585339.8ca644d2cada6dc44b363b7383dcd4fe., hostname=bd6592c37bb2,40087,1732486423713, seqNum=5 because the exception is null or not the one we care about 2024-11-24T22:16:48,092 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732486585339.8ca644d2cada6dc44b363b7383dcd4fe., hostname=bd6592c37bb2,40087,1732486423713, seqNum=5 , the old value is region=TestAcidGuarantees,,1732486585339.8ca644d2cada6dc44b363b7383dcd4fe., hostname=bd6592c37bb2,40087,1732486423713, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8ca644d2cada6dc44b363b7383dcd4fe, server=bd6592c37bb2,40087,1732486423713 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T22:16:48,092 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732486585339.8ca644d2cada6dc44b363b7383dcd4fe., hostname=bd6592c37bb2,40087,1732486423713, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8ca644d2cada6dc44b363b7383dcd4fe, server=bd6592c37bb2,40087,1732486423713 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T22:16:48,092 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732486585339.8ca644d2cada6dc44b363b7383dcd4fe., hostname=bd6592c37bb2,40087,1732486423713, seqNum=5 because the exception is null or not the one we care about 2024-11-24T22:16:48,093 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40087 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8ca644d2cada6dc44b363b7383dcd4fe, server=bd6592c37bb2,40087,1732486423713 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-24T22:16:48,093 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40087 {}] ipc.CallRunner(138): callId: 211 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49060 deadline: 1732486668086, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8ca644d2cada6dc44b363b7383dcd4fe, server=bd6592c37bb2,40087,1732486423713 2024-11-24T22:16:48,094 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732486585339.8ca644d2cada6dc44b363b7383dcd4fe., hostname=bd6592c37bb2,40087,1732486423713, seqNum=5 , the old value is region=TestAcidGuarantees,,1732486585339.8ca644d2cada6dc44b363b7383dcd4fe., hostname=bd6592c37bb2,40087,1732486423713, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8ca644d2cada6dc44b363b7383dcd4fe, server=bd6592c37bb2,40087,1732486423713 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T22:16:48,094 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40087 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8ca644d2cada6dc44b363b7383dcd4fe, server=bd6592c37bb2,40087,1732486423713 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-24T22:16:48,094 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732486585339.8ca644d2cada6dc44b363b7383dcd4fe., hostname=bd6592c37bb2,40087,1732486423713, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8ca644d2cada6dc44b363b7383dcd4fe, server=bd6592c37bb2,40087,1732486423713 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T22:16:48,094 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732486585339.8ca644d2cada6dc44b363b7383dcd4fe., hostname=bd6592c37bb2,40087,1732486423713, seqNum=5 because the exception is null or not the one we care about 2024-11-24T22:16:48,094 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40087 {}] ipc.CallRunner(138): callId: 236 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48986 deadline: 1732486668086, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8ca644d2cada6dc44b363b7383dcd4fe, server=bd6592c37bb2,40087,1732486423713 2024-11-24T22:16:48,094 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732486585339.8ca644d2cada6dc44b363b7383dcd4fe., hostname=bd6592c37bb2,40087,1732486423713, seqNum=5 , the old value is region=TestAcidGuarantees,,1732486585339.8ca644d2cada6dc44b363b7383dcd4fe., hostname=bd6592c37bb2,40087,1732486423713, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8ca644d2cada6dc44b363b7383dcd4fe, server=bd6592c37bb2,40087,1732486423713 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T22:16:48,094 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732486585339.8ca644d2cada6dc44b363b7383dcd4fe., hostname=bd6592c37bb2,40087,1732486423713, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8ca644d2cada6dc44b363b7383dcd4fe, server=bd6592c37bb2,40087,1732486423713 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T22:16:48,094 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732486585339.8ca644d2cada6dc44b363b7383dcd4fe., hostname=bd6592c37bb2,40087,1732486423713, seqNum=5 because the exception is null or not the one we care about 2024-11-24T22:16:48,094 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40087 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8ca644d2cada6dc44b363b7383dcd4fe, server=bd6592c37bb2,40087,1732486423713 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-24T22:16:48,094 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40087 {}] ipc.CallRunner(138): callId: 153 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49026 deadline: 1732486668088, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8ca644d2cada6dc44b363b7383dcd4fe, server=bd6592c37bb2,40087,1732486423713 2024-11-24T22:16:48,094 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732486585339.8ca644d2cada6dc44b363b7383dcd4fe., hostname=bd6592c37bb2,40087,1732486423713, seqNum=5 , the old value is region=TestAcidGuarantees,,1732486585339.8ca644d2cada6dc44b363b7383dcd4fe., hostname=bd6592c37bb2,40087,1732486423713, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8ca644d2cada6dc44b363b7383dcd4fe, server=bd6592c37bb2,40087,1732486423713 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T22:16:48,094 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732486585339.8ca644d2cada6dc44b363b7383dcd4fe., hostname=bd6592c37bb2,40087,1732486423713, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8ca644d2cada6dc44b363b7383dcd4fe, server=bd6592c37bb2,40087,1732486423713 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T22:16:48,094 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732486585339.8ca644d2cada6dc44b363b7383dcd4fe., hostname=bd6592c37bb2,40087,1732486423713, seqNum=5 because the exception is null or not the one we care about 2024-11-24T22:16:48,097 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45929 is added to blk_1073742479_1655 (size=13085) 2024-11-24T22:16:48,102 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45929 is added to blk_1073742480_1656 (size=4469) 2024-11-24T22:16:48,103 INFO [RS:0;bd6592c37bb2:40087-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 8ca644d2cada6dc44b363b7383dcd4fe#A#compaction#549 average throughput is 1.29 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-24T22:16:48,103 DEBUG [RS:0;bd6592c37bb2:40087-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/8ca644d2cada6dc44b363b7383dcd4fe/.tmp/A/6b8970e0d95a497095d0ced45e8bb8b2 is 175, key is test_row_0/A:col10/1732486607244/Put/seqid=0 2024-11-24T22:16:48,107 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45929 is added to blk_1073742481_1657 (size=32039) 2024-11-24T22:16:48,111 DEBUG [RS:0;bd6592c37bb2:40087-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/8ca644d2cada6dc44b363b7383dcd4fe/.tmp/A/6b8970e0d95a497095d0ced45e8bb8b2 as hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/8ca644d2cada6dc44b363b7383dcd4fe/A/6b8970e0d95a497095d0ced45e8bb8b2 2024-11-24T22:16:48,114 INFO [RS:0;bd6592c37bb2:40087-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 8ca644d2cada6dc44b363b7383dcd4fe/A of 8ca644d2cada6dc44b363b7383dcd4fe into 6b8970e0d95a497095d0ced45e8bb8b2(size=31.3 K), total size for store is 31.3 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-24T22:16:48,115 DEBUG [RS:0;bd6592c37bb2:40087-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 8ca644d2cada6dc44b363b7383dcd4fe: 2024-11-24T22:16:48,115 INFO [RS:0;bd6592c37bb2:40087-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732486585339.8ca644d2cada6dc44b363b7383dcd4fe., storeName=8ca644d2cada6dc44b363b7383dcd4fe/A, priority=12, startTime=1732486608072; duration=0sec 2024-11-24T22:16:48,115 DEBUG [RS:0;bd6592c37bb2:40087-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-24T22:16:48,115 DEBUG [RS:0;bd6592c37bb2:40087-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 8ca644d2cada6dc44b363b7383dcd4fe:A 2024-11-24T22:16:48,115 DEBUG [RS:0;bd6592c37bb2:40087-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-24T22:16:48,116 DEBUG [RS:0;bd6592c37bb2:40087-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 49852 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-24T22:16:48,116 DEBUG [RS:0;bd6592c37bb2:40087-shortCompactions-0 {}] regionserver.HStore(1541): 8ca644d2cada6dc44b363b7383dcd4fe/C is initiating minor compaction (all files) 2024-11-24T22:16:48,116 INFO [RS:0;bd6592c37bb2:40087-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 8ca644d2cada6dc44b363b7383dcd4fe/C in TestAcidGuarantees,,1732486585339.8ca644d2cada6dc44b363b7383dcd4fe. 2024-11-24T22:16:48,116 INFO [RS:0;bd6592c37bb2:40087-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/8ca644d2cada6dc44b363b7383dcd4fe/C/ae16909713a3403090f5a0ac150c170a, hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/8ca644d2cada6dc44b363b7383dcd4fe/C/d5f5e7c1ceb141538808728969043dc6, hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/8ca644d2cada6dc44b363b7383dcd4fe/C/2795ef0eb8634c1a81eabaac53d9372a, hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/8ca644d2cada6dc44b363b7383dcd4fe/C/40a953156e07441eb8fc6867ae9b1954] into tmpdir=hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/8ca644d2cada6dc44b363b7383dcd4fe/.tmp, totalSize=48.7 K 2024-11-24T22:16:48,116 DEBUG [RS:0;bd6592c37bb2:40087-shortCompactions-0 {}] compactions.Compactor(225): Compacting ae16909713a3403090f5a0ac150c170a, keycount=150, bloomtype=ROW, size=12.6 K, encoding=NONE, compression=NONE, seqNum=744, earliestPutTs=1732486604909 2024-11-24T22:16:48,116 DEBUG [RS:0;bd6592c37bb2:40087-shortCompactions-0 {}] compactions.Compactor(225): Compacting d5f5e7c1ceb141538808728969043dc6, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=777, earliestPutTs=1732486606021 2024-11-24T22:16:48,116 DEBUG [RS:0;bd6592c37bb2:40087-shortCompactions-0 {}] compactions.Compactor(225): Compacting 2795ef0eb8634c1a81eabaac53d9372a, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=793, earliestPutTs=1732486606060 2024-11-24T22:16:48,116 DEBUG [RS:0;bd6592c37bb2:40087-shortCompactions-0 {}] compactions.Compactor(225): Compacting 40a953156e07441eb8fc6867ae9b1954, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=815, earliestPutTs=1732486607244 2024-11-24T22:16:48,124 INFO [RS:0;bd6592c37bb2:40087-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 8ca644d2cada6dc44b363b7383dcd4fe#C#compaction#551 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-24T22:16:48,124 DEBUG [RS:0;bd6592c37bb2:40087-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/8ca644d2cada6dc44b363b7383dcd4fe/.tmp/C/0f7897c4877f4efbbdefd8c788913552 is 50, key is test_row_0/C:col10/1732486607244/Put/seqid=0 2024-11-24T22:16:48,128 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45929 is added to blk_1073742482_1658 (size=13085) 2024-11-24T22:16:48,135 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=40087 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=167 2024-11-24T22:16:48,135 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bd6592c37bb2:0-2 {event_type=RS_FLUSH_REGIONS, pid=167}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732486585339.8ca644d2cada6dc44b363b7383dcd4fe. 2024-11-24T22:16:48,135 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bd6592c37bb2:0-2 {event_type=RS_FLUSH_REGIONS, pid=167}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732486585339.8ca644d2cada6dc44b363b7383dcd4fe. as already flushing 2024-11-24T22:16:48,135 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bd6592c37bb2:0-2 {event_type=RS_FLUSH_REGIONS, pid=167}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732486585339.8ca644d2cada6dc44b363b7383dcd4fe. 2024-11-24T22:16:48,135 ERROR [RS_FLUSH_OPERATIONS-regionserver/bd6592c37bb2:0-2 {event_type=RS_FLUSH_REGIONS, pid=167}] handler.RSProcedureHandler(58): pid=167 java.io.IOException: Unable to complete flush {ENCODED => 8ca644d2cada6dc44b363b7383dcd4fe, NAME => 'TestAcidGuarantees,,1732486585339.8ca644d2cada6dc44b363b7383dcd4fe.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T22:16:48,135 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bd6592c37bb2:0-2 {event_type=RS_FLUSH_REGIONS, pid=167}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=167 java.io.IOException: Unable to complete flush {ENCODED => 8ca644d2cada6dc44b363b7383dcd4fe, NAME => 'TestAcidGuarantees,,1732486585339.8ca644d2cada6dc44b363b7383dcd4fe.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T22:16:48,136 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37701 {}] master.HMaster(4177): Remote procedure failed, pid=167 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 8ca644d2cada6dc44b363b7383dcd4fe, NAME => 'TestAcidGuarantees,,1732486585339.8ca644d2cada6dc44b363b7383dcd4fe.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 8ca644d2cada6dc44b363b7383dcd4fe, NAME => 'TestAcidGuarantees,,1732486585339.8ca644d2cada6dc44b363b7383dcd4fe.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T22:16:48,151 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37701 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=166 2024-11-24T22:16:48,203 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40087 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8ca644d2cada6dc44b363b7383dcd4fe, server=bd6592c37bb2,40087,1732486423713 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-24T22:16:48,203 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40087 {}] ipc.CallRunner(138): callId: 276 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49002 deadline: 1732486668200, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8ca644d2cada6dc44b363b7383dcd4fe, server=bd6592c37bb2,40087,1732486423713 2024-11-24T22:16:48,203 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40087 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8ca644d2cada6dc44b363b7383dcd4fe, server=bd6592c37bb2,40087,1732486423713 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-24T22:16:48,203 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732486585339.8ca644d2cada6dc44b363b7383dcd4fe., hostname=bd6592c37bb2,40087,1732486423713, seqNum=5 , the old value is region=TestAcidGuarantees,,1732486585339.8ca644d2cada6dc44b363b7383dcd4fe., hostname=bd6592c37bb2,40087,1732486423713, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8ca644d2cada6dc44b363b7383dcd4fe, server=bd6592c37bb2,40087,1732486423713 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T22:16:48,203 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732486585339.8ca644d2cada6dc44b363b7383dcd4fe., hostname=bd6592c37bb2,40087,1732486423713, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8ca644d2cada6dc44b363b7383dcd4fe, server=bd6592c37bb2,40087,1732486423713 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T22:16:48,203 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40087 {}] ipc.CallRunner(138): callId: 279 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49050 deadline: 1732486668200, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8ca644d2cada6dc44b363b7383dcd4fe, server=bd6592c37bb2,40087,1732486423713 2024-11-24T22:16:48,203 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732486585339.8ca644d2cada6dc44b363b7383dcd4fe., hostname=bd6592c37bb2,40087,1732486423713, seqNum=5 because the exception is null or not the one we care about 2024-11-24T22:16:48,204 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40087 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8ca644d2cada6dc44b363b7383dcd4fe, server=bd6592c37bb2,40087,1732486423713 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-24T22:16:48,204 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732486585339.8ca644d2cada6dc44b363b7383dcd4fe., hostname=bd6592c37bb2,40087,1732486423713, seqNum=5 , the old value is region=TestAcidGuarantees,,1732486585339.8ca644d2cada6dc44b363b7383dcd4fe., hostname=bd6592c37bb2,40087,1732486423713, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8ca644d2cada6dc44b363b7383dcd4fe, server=bd6592c37bb2,40087,1732486423713 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T22:16:48,204 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732486585339.8ca644d2cada6dc44b363b7383dcd4fe., hostname=bd6592c37bb2,40087,1732486423713, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8ca644d2cada6dc44b363b7383dcd4fe, server=bd6592c37bb2,40087,1732486423713 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T22:16:48,204 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40087 {}] ipc.CallRunner(138): callId: 237 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48986 deadline: 1732486668201, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8ca644d2cada6dc44b363b7383dcd4fe, server=bd6592c37bb2,40087,1732486423713 2024-11-24T22:16:48,204 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732486585339.8ca644d2cada6dc44b363b7383dcd4fe., hostname=bd6592c37bb2,40087,1732486423713, seqNum=5 because the exception is null or not the one we care about 2024-11-24T22:16:48,204 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40087 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8ca644d2cada6dc44b363b7383dcd4fe, server=bd6592c37bb2,40087,1732486423713 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-24T22:16:48,204 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732486585339.8ca644d2cada6dc44b363b7383dcd4fe., hostname=bd6592c37bb2,40087,1732486423713, seqNum=5 , the old value is region=TestAcidGuarantees,,1732486585339.8ca644d2cada6dc44b363b7383dcd4fe., hostname=bd6592c37bb2,40087,1732486423713, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8ca644d2cada6dc44b363b7383dcd4fe, server=bd6592c37bb2,40087,1732486423713 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T22:16:48,204 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732486585339.8ca644d2cada6dc44b363b7383dcd4fe., hostname=bd6592c37bb2,40087,1732486423713, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8ca644d2cada6dc44b363b7383dcd4fe, server=bd6592c37bb2,40087,1732486423713 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T22:16:48,204 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40087 {}] ipc.CallRunner(138): callId: 154 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49026 deadline: 1732486668201, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8ca644d2cada6dc44b363b7383dcd4fe, server=bd6592c37bb2,40087,1732486423713 2024-11-24T22:16:48,204 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732486585339.8ca644d2cada6dc44b363b7383dcd4fe., hostname=bd6592c37bb2,40087,1732486423713, seqNum=5 because the exception is null or not the one we care about 2024-11-24T22:16:48,204 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732486585339.8ca644d2cada6dc44b363b7383dcd4fe., hostname=bd6592c37bb2,40087,1732486423713, seqNum=5 , the old value is region=TestAcidGuarantees,,1732486585339.8ca644d2cada6dc44b363b7383dcd4fe., hostname=bd6592c37bb2,40087,1732486423713, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8ca644d2cada6dc44b363b7383dcd4fe, server=bd6592c37bb2,40087,1732486423713 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T22:16:48,204 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40087 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8ca644d2cada6dc44b363b7383dcd4fe, server=bd6592c37bb2,40087,1732486423713 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-24T22:16:48,204 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732486585339.8ca644d2cada6dc44b363b7383dcd4fe., hostname=bd6592c37bb2,40087,1732486423713, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8ca644d2cada6dc44b363b7383dcd4fe, server=bd6592c37bb2,40087,1732486423713 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T22:16:48,204 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732486585339.8ca644d2cada6dc44b363b7383dcd4fe., hostname=bd6592c37bb2,40087,1732486423713, seqNum=5 because the exception is null or not the one we care about 2024-11-24T22:16:48,204 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40087 {}] ipc.CallRunner(138): callId: 212 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49060 deadline: 1732486668201, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8ca644d2cada6dc44b363b7383dcd4fe, server=bd6592c37bb2,40087,1732486423713 2024-11-24T22:16:48,205 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732486585339.8ca644d2cada6dc44b363b7383dcd4fe., hostname=bd6592c37bb2,40087,1732486423713, seqNum=5 , the old value is region=TestAcidGuarantees,,1732486585339.8ca644d2cada6dc44b363b7383dcd4fe., hostname=bd6592c37bb2,40087,1732486423713, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8ca644d2cada6dc44b363b7383dcd4fe, server=bd6592c37bb2,40087,1732486423713 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T22:16:48,205 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732486585339.8ca644d2cada6dc44b363b7383dcd4fe., hostname=bd6592c37bb2,40087,1732486423713, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8ca644d2cada6dc44b363b7383dcd4fe, server=bd6592c37bb2,40087,1732486423713 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T22:16:48,205 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732486585339.8ca644d2cada6dc44b363b7383dcd4fe., hostname=bd6592c37bb2,40087,1732486423713, seqNum=5 because the exception is null or not the one we care about 2024-11-24T22:16:48,287 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=40087 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=167 2024-11-24T22:16:48,287 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bd6592c37bb2:0-0 {event_type=RS_FLUSH_REGIONS, pid=167}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732486585339.8ca644d2cada6dc44b363b7383dcd4fe. 2024-11-24T22:16:48,287 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bd6592c37bb2:0-0 {event_type=RS_FLUSH_REGIONS, pid=167}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732486585339.8ca644d2cada6dc44b363b7383dcd4fe. as already flushing 2024-11-24T22:16:48,287 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bd6592c37bb2:0-0 {event_type=RS_FLUSH_REGIONS, pid=167}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732486585339.8ca644d2cada6dc44b363b7383dcd4fe. 2024-11-24T22:16:48,288 ERROR [RS_FLUSH_OPERATIONS-regionserver/bd6592c37bb2:0-0 {event_type=RS_FLUSH_REGIONS, pid=167}] handler.RSProcedureHandler(58): pid=167 java.io.IOException: Unable to complete flush {ENCODED => 8ca644d2cada6dc44b363b7383dcd4fe, NAME => 'TestAcidGuarantees,,1732486585339.8ca644d2cada6dc44b363b7383dcd4fe.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T22:16:48,288 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bd6592c37bb2:0-0 {event_type=RS_FLUSH_REGIONS, pid=167}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=167 java.io.IOException: Unable to complete flush {ENCODED => 8ca644d2cada6dc44b363b7383dcd4fe, NAME => 'TestAcidGuarantees,,1732486585339.8ca644d2cada6dc44b363b7383dcd4fe.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T22:16:48,288 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37701 {}] master.HMaster(4177): Remote procedure failed, pid=167 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 8ca644d2cada6dc44b363b7383dcd4fe, NAME => 'TestAcidGuarantees,,1732486585339.8ca644d2cada6dc44b363b7383dcd4fe.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 8ca644d2cada6dc44b363b7383dcd4fe, NAME => 'TestAcidGuarantees,,1732486585339.8ca644d2cada6dc44b363b7383dcd4fe.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T22:16:48,414 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40087 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8ca644d2cada6dc44b363b7383dcd4fe, server=bd6592c37bb2,40087,1732486423713 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-24T22:16:48,415 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40087 {}] ipc.CallRunner(138): callId: 277 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49002 deadline: 1732486668410, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8ca644d2cada6dc44b363b7383dcd4fe, server=bd6592c37bb2,40087,1732486423713 2024-11-24T22:16:48,415 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40087 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8ca644d2cada6dc44b363b7383dcd4fe, server=bd6592c37bb2,40087,1732486423713 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-24T22:16:48,415 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732486585339.8ca644d2cada6dc44b363b7383dcd4fe., hostname=bd6592c37bb2,40087,1732486423713, seqNum=5 , the old value is region=TestAcidGuarantees,,1732486585339.8ca644d2cada6dc44b363b7383dcd4fe., hostname=bd6592c37bb2,40087,1732486423713, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8ca644d2cada6dc44b363b7383dcd4fe, server=bd6592c37bb2,40087,1732486423713 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T22:16:48,415 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732486585339.8ca644d2cada6dc44b363b7383dcd4fe., hostname=bd6592c37bb2,40087,1732486423713, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8ca644d2cada6dc44b363b7383dcd4fe, server=bd6592c37bb2,40087,1732486423713 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T22:16:48,415 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40087 {}] ipc.CallRunner(138): callId: 280 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49050 deadline: 1732486668410, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8ca644d2cada6dc44b363b7383dcd4fe, server=bd6592c37bb2,40087,1732486423713 2024-11-24T22:16:48,415 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732486585339.8ca644d2cada6dc44b363b7383dcd4fe., hostname=bd6592c37bb2,40087,1732486423713, seqNum=5 because the exception is null or not the one we care about 2024-11-24T22:16:48,415 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40087 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8ca644d2cada6dc44b363b7383dcd4fe, server=bd6592c37bb2,40087,1732486423713 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-24T22:16:48,415 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732486585339.8ca644d2cada6dc44b363b7383dcd4fe., hostname=bd6592c37bb2,40087,1732486423713, seqNum=5 , the old value is region=TestAcidGuarantees,,1732486585339.8ca644d2cada6dc44b363b7383dcd4fe., hostname=bd6592c37bb2,40087,1732486423713, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8ca644d2cada6dc44b363b7383dcd4fe, server=bd6592c37bb2,40087,1732486423713 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T22:16:48,415 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732486585339.8ca644d2cada6dc44b363b7383dcd4fe., hostname=bd6592c37bb2,40087,1732486423713, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8ca644d2cada6dc44b363b7383dcd4fe, server=bd6592c37bb2,40087,1732486423713 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T22:16:48,416 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40087 {}] ipc.CallRunner(138): callId: 238 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48986 deadline: 1732486668411, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8ca644d2cada6dc44b363b7383dcd4fe, server=bd6592c37bb2,40087,1732486423713 2024-11-24T22:16:48,416 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732486585339.8ca644d2cada6dc44b363b7383dcd4fe., hostname=bd6592c37bb2,40087,1732486423713, seqNum=5 because the exception is null or not the one we care about 2024-11-24T22:16:48,416 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732486585339.8ca644d2cada6dc44b363b7383dcd4fe., hostname=bd6592c37bb2,40087,1732486423713, seqNum=5 , the old value is region=TestAcidGuarantees,,1732486585339.8ca644d2cada6dc44b363b7383dcd4fe., hostname=bd6592c37bb2,40087,1732486423713, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8ca644d2cada6dc44b363b7383dcd4fe, server=bd6592c37bb2,40087,1732486423713 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T22:16:48,416 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732486585339.8ca644d2cada6dc44b363b7383dcd4fe., hostname=bd6592c37bb2,40087,1732486423713, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8ca644d2cada6dc44b363b7383dcd4fe, server=bd6592c37bb2,40087,1732486423713 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T22:16:48,416 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40087 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8ca644d2cada6dc44b363b7383dcd4fe, server=bd6592c37bb2,40087,1732486423713 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-24T22:16:48,416 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732486585339.8ca644d2cada6dc44b363b7383dcd4fe., hostname=bd6592c37bb2,40087,1732486423713, seqNum=5 because the exception is null or not the one we care about 2024-11-24T22:16:48,416 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40087 {}] ipc.CallRunner(138): callId: 155 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49026 deadline: 1732486668411, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8ca644d2cada6dc44b363b7383dcd4fe, server=bd6592c37bb2,40087,1732486423713 2024-11-24T22:16:48,416 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732486585339.8ca644d2cada6dc44b363b7383dcd4fe., hostname=bd6592c37bb2,40087,1732486423713, seqNum=5 , the old value is region=TestAcidGuarantees,,1732486585339.8ca644d2cada6dc44b363b7383dcd4fe., hostname=bd6592c37bb2,40087,1732486423713, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8ca644d2cada6dc44b363b7383dcd4fe, server=bd6592c37bb2,40087,1732486423713 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T22:16:48,416 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732486585339.8ca644d2cada6dc44b363b7383dcd4fe., hostname=bd6592c37bb2,40087,1732486423713, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8ca644d2cada6dc44b363b7383dcd4fe, server=bd6592c37bb2,40087,1732486423713 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T22:16:48,416 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732486585339.8ca644d2cada6dc44b363b7383dcd4fe., hostname=bd6592c37bb2,40087,1732486423713, seqNum=5 because the exception is null or not the one we care about 2024-11-24T22:16:48,416 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40087 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8ca644d2cada6dc44b363b7383dcd4fe, server=bd6592c37bb2,40087,1732486423713 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-24T22:16:48,416 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40087 {}] ipc.CallRunner(138): callId: 213 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49060 deadline: 1732486668411, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8ca644d2cada6dc44b363b7383dcd4fe, server=bd6592c37bb2,40087,1732486423713 2024-11-24T22:16:48,416 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732486585339.8ca644d2cada6dc44b363b7383dcd4fe., hostname=bd6592c37bb2,40087,1732486423713, seqNum=5 , the old value is region=TestAcidGuarantees,,1732486585339.8ca644d2cada6dc44b363b7383dcd4fe., hostname=bd6592c37bb2,40087,1732486423713, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8ca644d2cada6dc44b363b7383dcd4fe, server=bd6592c37bb2,40087,1732486423713 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T22:16:48,416 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732486585339.8ca644d2cada6dc44b363b7383dcd4fe., hostname=bd6592c37bb2,40087,1732486423713, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8ca644d2cada6dc44b363b7383dcd4fe, server=bd6592c37bb2,40087,1732486423713 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T22:16:48,417 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732486585339.8ca644d2cada6dc44b363b7383dcd4fe., hostname=bd6592c37bb2,40087,1732486423713, seqNum=5 because the exception is null or not the one we care about 2024-11-24T22:16:48,440 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=40087 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=167 2024-11-24T22:16:48,440 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bd6592c37bb2:0-1 {event_type=RS_FLUSH_REGIONS, pid=167}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732486585339.8ca644d2cada6dc44b363b7383dcd4fe. 2024-11-24T22:16:48,440 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bd6592c37bb2:0-1 {event_type=RS_FLUSH_REGIONS, pid=167}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732486585339.8ca644d2cada6dc44b363b7383dcd4fe. as already flushing 2024-11-24T22:16:48,440 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bd6592c37bb2:0-1 {event_type=RS_FLUSH_REGIONS, pid=167}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732486585339.8ca644d2cada6dc44b363b7383dcd4fe. 2024-11-24T22:16:48,440 ERROR [RS_FLUSH_OPERATIONS-regionserver/bd6592c37bb2:0-1 {event_type=RS_FLUSH_REGIONS, pid=167}] handler.RSProcedureHandler(58): pid=167 java.io.IOException: Unable to complete flush {ENCODED => 8ca644d2cada6dc44b363b7383dcd4fe, NAME => 'TestAcidGuarantees,,1732486585339.8ca644d2cada6dc44b363b7383dcd4fe.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T22:16:48,440 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bd6592c37bb2:0-1 {event_type=RS_FLUSH_REGIONS, pid=167}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=167 java.io.IOException: Unable to complete flush {ENCODED => 8ca644d2cada6dc44b363b7383dcd4fe, NAME => 'TestAcidGuarantees,,1732486585339.8ca644d2cada6dc44b363b7383dcd4fe.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T22:16:48,441 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37701 {}] master.HMaster(4177): Remote procedure failed, pid=167 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 8ca644d2cada6dc44b363b7383dcd4fe, NAME => 'TestAcidGuarantees,,1732486585339.8ca644d2cada6dc44b363b7383dcd4fe.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 8ca644d2cada6dc44b363b7383dcd4fe, NAME => 'TestAcidGuarantees,,1732486585339.8ca644d2cada6dc44b363b7383dcd4fe.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T22:16:48,454 INFO [Thread-2507 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-2507. 2024-11-24T22:16:48,454 DEBUG [Thread-2507 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicGetReader.workDone(AcidGuaranteesTestTool.java:234) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-24T22:16:48,454 DEBUG [Thread-2507 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-24T22:16:48,454 DEBUG [Thread-2507 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-24T22:16:48,455 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-24T22:16:48,456 INFO [Thread-2509 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-2509. 2024-11-24T22:16:48,456 DEBUG [Thread-2509 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicGetReader.workDone(AcidGuaranteesTestTool.java:234) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-24T22:16:48,456 DEBUG [Thread-2509 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-24T22:16:48,456 DEBUG [Thread-2509 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-24T22:16:48,457 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-24T22:16:48,458 INFO [Thread-2505 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-2505. 2024-11-24T22:16:48,458 DEBUG [Thread-2505 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicGetReader.workDone(AcidGuaranteesTestTool.java:234) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-24T22:16:48,458 DEBUG [Thread-2505 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-24T22:16:48,458 DEBUG [Thread-2505 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-24T22:16:48,458 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-24T22:16:48,459 INFO [Thread-2511 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-2511. 2024-11-24T22:16:48,459 DEBUG [Thread-2511 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicGetReader.workDone(AcidGuaranteesTestTool.java:234) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-24T22:16:48,459 DEBUG [Thread-2511 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-24T22:16:48,459 DEBUG [Thread-2511 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-24T22:16:48,459 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-24T22:16:48,460 INFO [Thread-2513 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-2513. 2024-11-24T22:16:48,460 DEBUG [Thread-2513 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicGetReader.workDone(AcidGuaranteesTestTool.java:234) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-24T22:16:48,460 DEBUG [Thread-2513 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-24T22:16:48,460 DEBUG [Thread-2513 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-24T22:16:48,460 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-24T22:16:48,490 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T22:16:48,493 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241124bda66f2b082942a998944897f1893b7f_8ca644d2cada6dc44b363b7383dcd4fe to hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241124bda66f2b082942a998944897f1893b7f_8ca644d2cada6dc44b363b7383dcd4fe 2024-11-24T22:16:48,493 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/8ca644d2cada6dc44b363b7383dcd4fe/.tmp/A/3589d762efd14116bfc39bff83dbd9d9, store: [table=TestAcidGuarantees family=A region=8ca644d2cada6dc44b363b7383dcd4fe] 2024-11-24T22:16:48,494 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/8ca644d2cada6dc44b363b7383dcd4fe/.tmp/A/3589d762efd14116bfc39bff83dbd9d9 is 175, key is test_row_0/A:col10/1732486608072/Put/seqid=0 2024-11-24T22:16:48,496 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45929 is added to blk_1073742483_1659 (size=39949) 2024-11-24T22:16:48,500 DEBUG [RS:0;bd6592c37bb2:40087-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/8ca644d2cada6dc44b363b7383dcd4fe/.tmp/B/ed0b458c5c334334a162cfbf3d0063e1 as hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/8ca644d2cada6dc44b363b7383dcd4fe/B/ed0b458c5c334334a162cfbf3d0063e1 2024-11-24T22:16:48,503 INFO [RS:0;bd6592c37bb2:40087-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 8ca644d2cada6dc44b363b7383dcd4fe/B of 8ca644d2cada6dc44b363b7383dcd4fe into ed0b458c5c334334a162cfbf3d0063e1(size=12.8 K), total size for store is 12.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-24T22:16:48,503 DEBUG [RS:0;bd6592c37bb2:40087-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 8ca644d2cada6dc44b363b7383dcd4fe: 2024-11-24T22:16:48,503 INFO [RS:0;bd6592c37bb2:40087-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732486585339.8ca644d2cada6dc44b363b7383dcd4fe., storeName=8ca644d2cada6dc44b363b7383dcd4fe/B, priority=12, startTime=1732486608073; duration=0sec 2024-11-24T22:16:48,503 DEBUG [RS:0;bd6592c37bb2:40087-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-24T22:16:48,503 DEBUG [RS:0;bd6592c37bb2:40087-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 8ca644d2cada6dc44b363b7383dcd4fe:B 2024-11-24T22:16:48,532 DEBUG [RS:0;bd6592c37bb2:40087-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/8ca644d2cada6dc44b363b7383dcd4fe/.tmp/C/0f7897c4877f4efbbdefd8c788913552 as hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/8ca644d2cada6dc44b363b7383dcd4fe/C/0f7897c4877f4efbbdefd8c788913552 2024-11-24T22:16:48,535 INFO [RS:0;bd6592c37bb2:40087-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 8ca644d2cada6dc44b363b7383dcd4fe/C of 8ca644d2cada6dc44b363b7383dcd4fe into 0f7897c4877f4efbbdefd8c788913552(size=12.8 K), total size for store is 12.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-24T22:16:48,535 DEBUG [RS:0;bd6592c37bb2:40087-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 8ca644d2cada6dc44b363b7383dcd4fe: 2024-11-24T22:16:48,535 INFO [RS:0;bd6592c37bb2:40087-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732486585339.8ca644d2cada6dc44b363b7383dcd4fe., storeName=8ca644d2cada6dc44b363b7383dcd4fe/C, priority=12, startTime=1732486608073; duration=0sec 2024-11-24T22:16:48,535 DEBUG [RS:0;bd6592c37bb2:40087-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-24T22:16:48,535 DEBUG [RS:0;bd6592c37bb2:40087-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 8ca644d2cada6dc44b363b7383dcd4fe:C 2024-11-24T22:16:48,593 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=40087 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=167 2024-11-24T22:16:48,593 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bd6592c37bb2:0-2 {event_type=RS_FLUSH_REGIONS, pid=167}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732486585339.8ca644d2cada6dc44b363b7383dcd4fe. 2024-11-24T22:16:48,593 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bd6592c37bb2:0-2 {event_type=RS_FLUSH_REGIONS, pid=167}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732486585339.8ca644d2cada6dc44b363b7383dcd4fe. as already flushing 2024-11-24T22:16:48,593 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bd6592c37bb2:0-2 {event_type=RS_FLUSH_REGIONS, pid=167}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732486585339.8ca644d2cada6dc44b363b7383dcd4fe. 2024-11-24T22:16:48,593 ERROR [RS_FLUSH_OPERATIONS-regionserver/bd6592c37bb2:0-2 {event_type=RS_FLUSH_REGIONS, pid=167}] handler.RSProcedureHandler(58): pid=167 java.io.IOException: Unable to complete flush {ENCODED => 8ca644d2cada6dc44b363b7383dcd4fe, NAME => 'TestAcidGuarantees,,1732486585339.8ca644d2cada6dc44b363b7383dcd4fe.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T22:16:48,593 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bd6592c37bb2:0-2 {event_type=RS_FLUSH_REGIONS, pid=167}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=167 java.io.IOException: Unable to complete flush {ENCODED => 8ca644d2cada6dc44b363b7383dcd4fe, NAME => 'TestAcidGuarantees,,1732486585339.8ca644d2cada6dc44b363b7383dcd4fe.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T22:16:48,594 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37701 {}] master.HMaster(4177): Remote procedure failed, pid=167 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 8ca644d2cada6dc44b363b7383dcd4fe, NAME => 'TestAcidGuarantees,,1732486585339.8ca644d2cada6dc44b363b7383dcd4fe.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 8ca644d2cada6dc44b363b7383dcd4fe, NAME => 'TestAcidGuarantees,,1732486585339.8ca644d2cada6dc44b363b7383dcd4fe.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T22:16:48,661 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37701 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=166 2024-11-24T22:16:48,721 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40087 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8ca644d2cada6dc44b363b7383dcd4fe, server=bd6592c37bb2,40087,1732486423713 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-24T22:16:48,721 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40087 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8ca644d2cada6dc44b363b7383dcd4fe, server=bd6592c37bb2,40087,1732486423713 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-24T22:16:48,721 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40087 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8ca644d2cada6dc44b363b7383dcd4fe, server=bd6592c37bb2,40087,1732486423713 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-24T22:16:48,721 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40087 {}] ipc.CallRunner(138): callId: 239 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48986 deadline: 1732486668721, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8ca644d2cada6dc44b363b7383dcd4fe, server=bd6592c37bb2,40087,1732486423713 2024-11-24T22:16:48,721 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40087 {}] ipc.CallRunner(138): callId: 281 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49050 deadline: 1732486668720, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8ca644d2cada6dc44b363b7383dcd4fe, server=bd6592c37bb2,40087,1732486423713 2024-11-24T22:16:48,721 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40087 {}] ipc.CallRunner(138): callId: 278 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49002 deadline: 1732486668720, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8ca644d2cada6dc44b363b7383dcd4fe, server=bd6592c37bb2,40087,1732486423713 2024-11-24T22:16:48,721 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40087 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8ca644d2cada6dc44b363b7383dcd4fe, server=bd6592c37bb2,40087,1732486423713 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-24T22:16:48,721 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732486585339.8ca644d2cada6dc44b363b7383dcd4fe., hostname=bd6592c37bb2,40087,1732486423713, seqNum=5 , the old value is region=TestAcidGuarantees,,1732486585339.8ca644d2cada6dc44b363b7383dcd4fe., hostname=bd6592c37bb2,40087,1732486423713, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8ca644d2cada6dc44b363b7383dcd4fe, server=bd6592c37bb2,40087,1732486423713 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T22:16:48,721 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40087 {}] ipc.CallRunner(138): callId: 156 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49026 deadline: 1732486668721, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8ca644d2cada6dc44b363b7383dcd4fe, server=bd6592c37bb2,40087,1732486423713 2024-11-24T22:16:48,721 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732486585339.8ca644d2cada6dc44b363b7383dcd4fe., hostname=bd6592c37bb2,40087,1732486423713, seqNum=5 , the old value is region=TestAcidGuarantees,,1732486585339.8ca644d2cada6dc44b363b7383dcd4fe., hostname=bd6592c37bb2,40087,1732486423713, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8ca644d2cada6dc44b363b7383dcd4fe, server=bd6592c37bb2,40087,1732486423713 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T22:16:48,721 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40087 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8ca644d2cada6dc44b363b7383dcd4fe, server=bd6592c37bb2,40087,1732486423713 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-24T22:16:48,721 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732486585339.8ca644d2cada6dc44b363b7383dcd4fe., hostname=bd6592c37bb2,40087,1732486423713, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8ca644d2cada6dc44b363b7383dcd4fe, server=bd6592c37bb2,40087,1732486423713 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T22:16:48,721 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732486585339.8ca644d2cada6dc44b363b7383dcd4fe., hostname=bd6592c37bb2,40087,1732486423713, seqNum=5 , the old value is region=TestAcidGuarantees,,1732486585339.8ca644d2cada6dc44b363b7383dcd4fe., hostname=bd6592c37bb2,40087,1732486423713, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8ca644d2cada6dc44b363b7383dcd4fe, server=bd6592c37bb2,40087,1732486423713 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T22:16:48,721 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732486585339.8ca644d2cada6dc44b363b7383dcd4fe., hostname=bd6592c37bb2,40087,1732486423713, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8ca644d2cada6dc44b363b7383dcd4fe, server=bd6592c37bb2,40087,1732486423713 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T22:16:48,721 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732486585339.8ca644d2cada6dc44b363b7383dcd4fe., hostname=bd6592c37bb2,40087,1732486423713, seqNum=5 because the exception is null or not the one we care about 2024-11-24T22:16:48,721 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732486585339.8ca644d2cada6dc44b363b7383dcd4fe., hostname=bd6592c37bb2,40087,1732486423713, seqNum=5 because the exception is null or not the one we care about 2024-11-24T22:16:48,721 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732486585339.8ca644d2cada6dc44b363b7383dcd4fe., hostname=bd6592c37bb2,40087,1732486423713, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8ca644d2cada6dc44b363b7383dcd4fe, server=bd6592c37bb2,40087,1732486423713 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T22:16:48,721 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40087 {}] ipc.CallRunner(138): callId: 214 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49060 deadline: 1732486668721, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8ca644d2cada6dc44b363b7383dcd4fe, server=bd6592c37bb2,40087,1732486423713 2024-11-24T22:16:48,721 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732486585339.8ca644d2cada6dc44b363b7383dcd4fe., hostname=bd6592c37bb2,40087,1732486423713, seqNum=5 because the exception is null or not the one we care about 2024-11-24T22:16:48,721 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732486585339.8ca644d2cada6dc44b363b7383dcd4fe., hostname=bd6592c37bb2,40087,1732486423713, seqNum=5 , the old value is region=TestAcidGuarantees,,1732486585339.8ca644d2cada6dc44b363b7383dcd4fe., hostname=bd6592c37bb2,40087,1732486423713, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8ca644d2cada6dc44b363b7383dcd4fe, server=bd6592c37bb2,40087,1732486423713 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T22:16:48,721 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732486585339.8ca644d2cada6dc44b363b7383dcd4fe., hostname=bd6592c37bb2,40087,1732486423713, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8ca644d2cada6dc44b363b7383dcd4fe, server=bd6592c37bb2,40087,1732486423713 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T22:16:48,721 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732486585339.8ca644d2cada6dc44b363b7383dcd4fe., hostname=bd6592c37bb2,40087,1732486423713, seqNum=5 because the exception is null or not the one we care about 2024-11-24T22:16:48,721 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732486585339.8ca644d2cada6dc44b363b7383dcd4fe., hostname=bd6592c37bb2,40087,1732486423713, seqNum=5 , the old value is region=TestAcidGuarantees,,1732486585339.8ca644d2cada6dc44b363b7383dcd4fe., hostname=bd6592c37bb2,40087,1732486423713, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8ca644d2cada6dc44b363b7383dcd4fe, server=bd6592c37bb2,40087,1732486423713 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T22:16:48,722 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732486585339.8ca644d2cada6dc44b363b7383dcd4fe., hostname=bd6592c37bb2,40087,1732486423713, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8ca644d2cada6dc44b363b7383dcd4fe, server=bd6592c37bb2,40087,1732486423713 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T22:16:48,722 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732486585339.8ca644d2cada6dc44b363b7383dcd4fe., hostname=bd6592c37bb2,40087,1732486423713, seqNum=5 because the exception is null or not the one we care about 2024-11-24T22:16:48,745 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=40087 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=167 2024-11-24T22:16:48,746 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bd6592c37bb2:0-0 {event_type=RS_FLUSH_REGIONS, pid=167}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732486585339.8ca644d2cada6dc44b363b7383dcd4fe. 2024-11-24T22:16:48,746 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bd6592c37bb2:0-0 {event_type=RS_FLUSH_REGIONS, pid=167}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732486585339.8ca644d2cada6dc44b363b7383dcd4fe. as already flushing 2024-11-24T22:16:48,746 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bd6592c37bb2:0-0 {event_type=RS_FLUSH_REGIONS, pid=167}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732486585339.8ca644d2cada6dc44b363b7383dcd4fe. 2024-11-24T22:16:48,746 ERROR [RS_FLUSH_OPERATIONS-regionserver/bd6592c37bb2:0-0 {event_type=RS_FLUSH_REGIONS, pid=167}] handler.RSProcedureHandler(58): pid=167 java.io.IOException: Unable to complete flush {ENCODED => 8ca644d2cada6dc44b363b7383dcd4fe, NAME => 'TestAcidGuarantees,,1732486585339.8ca644d2cada6dc44b363b7383dcd4fe.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T22:16:48,746 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bd6592c37bb2:0-0 {event_type=RS_FLUSH_REGIONS, pid=167}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=167 java.io.IOException: Unable to complete flush {ENCODED => 8ca644d2cada6dc44b363b7383dcd4fe, NAME => 'TestAcidGuarantees,,1732486585339.8ca644d2cada6dc44b363b7383dcd4fe.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T22:16:48,746 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37701 {}] master.HMaster(4177): Remote procedure failed, pid=167 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 8ca644d2cada6dc44b363b7383dcd4fe, NAME => 'TestAcidGuarantees,,1732486585339.8ca644d2cada6dc44b363b7383dcd4fe.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 8ca644d2cada6dc44b363b7383dcd4fe, NAME => 'TestAcidGuarantees,,1732486585339.8ca644d2cada6dc44b363b7383dcd4fe.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T22:16:48,897 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=1005, memsize=44.7 K, hasBloomFilter=true, into tmp file hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/8ca644d2cada6dc44b363b7383dcd4fe/.tmp/A/3589d762efd14116bfc39bff83dbd9d9 2024-11-24T22:16:48,898 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=40087 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=167 2024-11-24T22:16:48,898 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bd6592c37bb2:0-1 {event_type=RS_FLUSH_REGIONS, pid=167}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732486585339.8ca644d2cada6dc44b363b7383dcd4fe. 2024-11-24T22:16:48,898 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bd6592c37bb2:0-1 {event_type=RS_FLUSH_REGIONS, pid=167}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732486585339.8ca644d2cada6dc44b363b7383dcd4fe. as already flushing 2024-11-24T22:16:48,898 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bd6592c37bb2:0-1 {event_type=RS_FLUSH_REGIONS, pid=167}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732486585339.8ca644d2cada6dc44b363b7383dcd4fe. 2024-11-24T22:16:48,898 ERROR [RS_FLUSH_OPERATIONS-regionserver/bd6592c37bb2:0-1 {event_type=RS_FLUSH_REGIONS, pid=167}] handler.RSProcedureHandler(58): pid=167 java.io.IOException: Unable to complete flush {ENCODED => 8ca644d2cada6dc44b363b7383dcd4fe, NAME => 'TestAcidGuarantees,,1732486585339.8ca644d2cada6dc44b363b7383dcd4fe.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T22:16:48,898 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bd6592c37bb2:0-1 {event_type=RS_FLUSH_REGIONS, pid=167}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=167 java.io.IOException: Unable to complete flush {ENCODED => 8ca644d2cada6dc44b363b7383dcd4fe, NAME => 'TestAcidGuarantees,,1732486585339.8ca644d2cada6dc44b363b7383dcd4fe.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T22:16:48,899 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37701 {}] master.HMaster(4177): Remote procedure failed, pid=167 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 8ca644d2cada6dc44b363b7383dcd4fe, NAME => 'TestAcidGuarantees,,1732486585339.8ca644d2cada6dc44b363b7383dcd4fe.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 8ca644d2cada6dc44b363b7383dcd4fe, NAME => 'TestAcidGuarantees,,1732486585339.8ca644d2cada6dc44b363b7383dcd4fe.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T22:16:48,903 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/8ca644d2cada6dc44b363b7383dcd4fe/.tmp/B/c6770c17f5d84d469ece27b38636bc53 is 50, key is test_row_0/B:col10/1732486608072/Put/seqid=0 2024-11-24T22:16:48,906 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45929 is added to blk_1073742484_1660 (size=12301) 2024-11-24T22:16:49,050 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=40087 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=167 2024-11-24T22:16:49,050 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bd6592c37bb2:0-2 {event_type=RS_FLUSH_REGIONS, pid=167}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732486585339.8ca644d2cada6dc44b363b7383dcd4fe. 2024-11-24T22:16:49,051 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bd6592c37bb2:0-2 {event_type=RS_FLUSH_REGIONS, pid=167}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732486585339.8ca644d2cada6dc44b363b7383dcd4fe. as already flushing 2024-11-24T22:16:49,051 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bd6592c37bb2:0-2 {event_type=RS_FLUSH_REGIONS, pid=167}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732486585339.8ca644d2cada6dc44b363b7383dcd4fe. 2024-11-24T22:16:49,051 ERROR [RS_FLUSH_OPERATIONS-regionserver/bd6592c37bb2:0-2 {event_type=RS_FLUSH_REGIONS, pid=167}] handler.RSProcedureHandler(58): pid=167 java.io.IOException: Unable to complete flush {ENCODED => 8ca644d2cada6dc44b363b7383dcd4fe, NAME => 'TestAcidGuarantees,,1732486585339.8ca644d2cada6dc44b363b7383dcd4fe.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T22:16:49,051 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bd6592c37bb2:0-2 {event_type=RS_FLUSH_REGIONS, pid=167}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=167 java.io.IOException: Unable to complete flush {ENCODED => 8ca644d2cada6dc44b363b7383dcd4fe, NAME => 'TestAcidGuarantees,,1732486585339.8ca644d2cada6dc44b363b7383dcd4fe.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T22:16:49,051 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37701 {}] master.HMaster(4177): Remote procedure failed, pid=167 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 8ca644d2cada6dc44b363b7383dcd4fe, NAME => 'TestAcidGuarantees,,1732486585339.8ca644d2cada6dc44b363b7383dcd4fe.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 8ca644d2cada6dc44b363b7383dcd4fe, NAME => 'TestAcidGuarantees,,1732486585339.8ca644d2cada6dc44b363b7383dcd4fe.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T22:16:49,203 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=40087 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=167 2024-11-24T22:16:49,203 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bd6592c37bb2:0-0 {event_type=RS_FLUSH_REGIONS, pid=167}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732486585339.8ca644d2cada6dc44b363b7383dcd4fe. 2024-11-24T22:16:49,203 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bd6592c37bb2:0-0 {event_type=RS_FLUSH_REGIONS, pid=167}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732486585339.8ca644d2cada6dc44b363b7383dcd4fe. as already flushing 2024-11-24T22:16:49,203 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bd6592c37bb2:0-0 {event_type=RS_FLUSH_REGIONS, pid=167}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732486585339.8ca644d2cada6dc44b363b7383dcd4fe. 2024-11-24T22:16:49,203 ERROR [RS_FLUSH_OPERATIONS-regionserver/bd6592c37bb2:0-0 {event_type=RS_FLUSH_REGIONS, pid=167}] handler.RSProcedureHandler(58): pid=167 java.io.IOException: Unable to complete flush {ENCODED => 8ca644d2cada6dc44b363b7383dcd4fe, NAME => 'TestAcidGuarantees,,1732486585339.8ca644d2cada6dc44b363b7383dcd4fe.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T22:16:49,203 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bd6592c37bb2:0-0 {event_type=RS_FLUSH_REGIONS, pid=167}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=167 java.io.IOException: Unable to complete flush {ENCODED => 8ca644d2cada6dc44b363b7383dcd4fe, NAME => 'TestAcidGuarantees,,1732486585339.8ca644d2cada6dc44b363b7383dcd4fe.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T22:16:49,204 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37701 {}] master.HMaster(4177): Remote procedure failed, pid=167 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 8ca644d2cada6dc44b363b7383dcd4fe, NAME => 'TestAcidGuarantees,,1732486585339.8ca644d2cada6dc44b363b7383dcd4fe.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 8ca644d2cada6dc44b363b7383dcd4fe, NAME => 'TestAcidGuarantees,,1732486585339.8ca644d2cada6dc44b363b7383dcd4fe.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T22:16:49,231 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40087 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8ca644d2cada6dc44b363b7383dcd4fe, server=bd6592c37bb2,40087,1732486423713 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-24T22:16:49,231 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40087 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8ca644d2cada6dc44b363b7383dcd4fe, server=bd6592c37bb2,40087,1732486423713 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-24T22:16:49,231 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40087 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8ca644d2cada6dc44b363b7383dcd4fe, server=bd6592c37bb2,40087,1732486423713 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-24T22:16:49,231 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40087 {}] ipc.CallRunner(138): callId: 279 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49002 deadline: 1732486669231, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8ca644d2cada6dc44b363b7383dcd4fe, server=bd6592c37bb2,40087,1732486423713 2024-11-24T22:16:49,232 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40087 {}] ipc.CallRunner(138): callId: 282 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49050 deadline: 1732486669231, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8ca644d2cada6dc44b363b7383dcd4fe, server=bd6592c37bb2,40087,1732486423713 2024-11-24T22:16:49,232 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40087 {}] ipc.CallRunner(138): callId: 240 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48986 deadline: 1732486669231, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8ca644d2cada6dc44b363b7383dcd4fe, server=bd6592c37bb2,40087,1732486423713 2024-11-24T22:16:49,232 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40087 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8ca644d2cada6dc44b363b7383dcd4fe, server=bd6592c37bb2,40087,1732486423713 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-24T22:16:49,232 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40087 {}] ipc.CallRunner(138): callId: 157 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49026 deadline: 1732486669231, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8ca644d2cada6dc44b363b7383dcd4fe, server=bd6592c37bb2,40087,1732486423713 2024-11-24T22:16:49,232 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40087 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8ca644d2cada6dc44b363b7383dcd4fe, server=bd6592c37bb2,40087,1732486423713 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-24T22:16:49,232 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732486585339.8ca644d2cada6dc44b363b7383dcd4fe., hostname=bd6592c37bb2,40087,1732486423713, seqNum=5 , the old value is region=TestAcidGuarantees,,1732486585339.8ca644d2cada6dc44b363b7383dcd4fe., hostname=bd6592c37bb2,40087,1732486423713, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8ca644d2cada6dc44b363b7383dcd4fe, server=bd6592c37bb2,40087,1732486423713 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T22:16:49,232 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732486585339.8ca644d2cada6dc44b363b7383dcd4fe., hostname=bd6592c37bb2,40087,1732486423713, seqNum=5 , the old value is region=TestAcidGuarantees,,1732486585339.8ca644d2cada6dc44b363b7383dcd4fe., hostname=bd6592c37bb2,40087,1732486423713, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8ca644d2cada6dc44b363b7383dcd4fe, server=bd6592c37bb2,40087,1732486423713 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T22:16:49,232 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732486585339.8ca644d2cada6dc44b363b7383dcd4fe., hostname=bd6592c37bb2,40087,1732486423713, seqNum=5 , the old value is region=TestAcidGuarantees,,1732486585339.8ca644d2cada6dc44b363b7383dcd4fe., hostname=bd6592c37bb2,40087,1732486423713, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8ca644d2cada6dc44b363b7383dcd4fe, server=bd6592c37bb2,40087,1732486423713 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T22:16:49,232 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732486585339.8ca644d2cada6dc44b363b7383dcd4fe., hostname=bd6592c37bb2,40087,1732486423713, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8ca644d2cada6dc44b363b7383dcd4fe, server=bd6592c37bb2,40087,1732486423713 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T22:16:49,232 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732486585339.8ca644d2cada6dc44b363b7383dcd4fe., hostname=bd6592c37bb2,40087,1732486423713, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8ca644d2cada6dc44b363b7383dcd4fe, server=bd6592c37bb2,40087,1732486423713 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T22:16:49,232 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732486585339.8ca644d2cada6dc44b363b7383dcd4fe., hostname=bd6592c37bb2,40087,1732486423713, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8ca644d2cada6dc44b363b7383dcd4fe, server=bd6592c37bb2,40087,1732486423713 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T22:16:49,232 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40087 {}] ipc.CallRunner(138): callId: 215 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49060 deadline: 1732486669231, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8ca644d2cada6dc44b363b7383dcd4fe, server=bd6592c37bb2,40087,1732486423713 2024-11-24T22:16:49,232 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732486585339.8ca644d2cada6dc44b363b7383dcd4fe., hostname=bd6592c37bb2,40087,1732486423713, seqNum=5 because the exception is null or not the one we care about 2024-11-24T22:16:49,232 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732486585339.8ca644d2cada6dc44b363b7383dcd4fe., hostname=bd6592c37bb2,40087,1732486423713, seqNum=5 because the exception is null or not the one we care about 2024-11-24T22:16:49,232 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732486585339.8ca644d2cada6dc44b363b7383dcd4fe., hostname=bd6592c37bb2,40087,1732486423713, seqNum=5 because the exception is null or not the one we care about 2024-11-24T22:16:49,232 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732486585339.8ca644d2cada6dc44b363b7383dcd4fe., hostname=bd6592c37bb2,40087,1732486423713, seqNum=5 , the old value is region=TestAcidGuarantees,,1732486585339.8ca644d2cada6dc44b363b7383dcd4fe., hostname=bd6592c37bb2,40087,1732486423713, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8ca644d2cada6dc44b363b7383dcd4fe, server=bd6592c37bb2,40087,1732486423713 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T22:16:49,232 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732486585339.8ca644d2cada6dc44b363b7383dcd4fe., hostname=bd6592c37bb2,40087,1732486423713, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8ca644d2cada6dc44b363b7383dcd4fe, server=bd6592c37bb2,40087,1732486423713 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T22:16:49,232 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732486585339.8ca644d2cada6dc44b363b7383dcd4fe., hostname=bd6592c37bb2,40087,1732486423713, seqNum=5 because the exception is null or not the one we care about 2024-11-24T22:16:49,232 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732486585339.8ca644d2cada6dc44b363b7383dcd4fe., hostname=bd6592c37bb2,40087,1732486423713, seqNum=5 , the old value is region=TestAcidGuarantees,,1732486585339.8ca644d2cada6dc44b363b7383dcd4fe., hostname=bd6592c37bb2,40087,1732486423713, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8ca644d2cada6dc44b363b7383dcd4fe, server=bd6592c37bb2,40087,1732486423713 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T22:16:49,232 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732486585339.8ca644d2cada6dc44b363b7383dcd4fe., hostname=bd6592c37bb2,40087,1732486423713, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8ca644d2cada6dc44b363b7383dcd4fe, server=bd6592c37bb2,40087,1732486423713 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T22:16:49,232 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732486585339.8ca644d2cada6dc44b363b7383dcd4fe., hostname=bd6592c37bb2,40087,1732486423713, seqNum=5 because the exception is null or not the one we care about 2024-11-24T22:16:49,306 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=1005 (bloomFilter=true), to=hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/8ca644d2cada6dc44b363b7383dcd4fe/.tmp/B/c6770c17f5d84d469ece27b38636bc53 2024-11-24T22:16:49,311 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/8ca644d2cada6dc44b363b7383dcd4fe/.tmp/C/90980d39589642b19b8884b4423eed49 is 50, key is test_row_0/C:col10/1732486608072/Put/seqid=0 2024-11-24T22:16:49,314 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45929 is added to blk_1073742485_1661 (size=12301) 2024-11-24T22:16:49,355 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=40087 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=167 2024-11-24T22:16:49,355 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bd6592c37bb2:0-1 {event_type=RS_FLUSH_REGIONS, pid=167}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732486585339.8ca644d2cada6dc44b363b7383dcd4fe. 2024-11-24T22:16:49,355 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bd6592c37bb2:0-1 {event_type=RS_FLUSH_REGIONS, pid=167}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732486585339.8ca644d2cada6dc44b363b7383dcd4fe. as already flushing 2024-11-24T22:16:49,355 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bd6592c37bb2:0-1 {event_type=RS_FLUSH_REGIONS, pid=167}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732486585339.8ca644d2cada6dc44b363b7383dcd4fe. 2024-11-24T22:16:49,355 ERROR [RS_FLUSH_OPERATIONS-regionserver/bd6592c37bb2:0-1 {event_type=RS_FLUSH_REGIONS, pid=167}] handler.RSProcedureHandler(58): pid=167 java.io.IOException: Unable to complete flush {ENCODED => 8ca644d2cada6dc44b363b7383dcd4fe, NAME => 'TestAcidGuarantees,,1732486585339.8ca644d2cada6dc44b363b7383dcd4fe.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T22:16:49,355 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bd6592c37bb2:0-1 {event_type=RS_FLUSH_REGIONS, pid=167}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=167 java.io.IOException: Unable to complete flush {ENCODED => 8ca644d2cada6dc44b363b7383dcd4fe, NAME => 'TestAcidGuarantees,,1732486585339.8ca644d2cada6dc44b363b7383dcd4fe.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T22:16:49,356 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37701 {}] master.HMaster(4177): Remote procedure failed, pid=167 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 8ca644d2cada6dc44b363b7383dcd4fe, NAME => 'TestAcidGuarantees,,1732486585339.8ca644d2cada6dc44b363b7383dcd4fe.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 8ca644d2cada6dc44b363b7383dcd4fe, NAME => 'TestAcidGuarantees,,1732486585339.8ca644d2cada6dc44b363b7383dcd4fe.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T22:16:49,507 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=40087 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=167 2024-11-24T22:16:49,508 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bd6592c37bb2:0-2 {event_type=RS_FLUSH_REGIONS, pid=167}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732486585339.8ca644d2cada6dc44b363b7383dcd4fe. 2024-11-24T22:16:49,508 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bd6592c37bb2:0-2 {event_type=RS_FLUSH_REGIONS, pid=167}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732486585339.8ca644d2cada6dc44b363b7383dcd4fe. as already flushing 2024-11-24T22:16:49,508 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bd6592c37bb2:0-2 {event_type=RS_FLUSH_REGIONS, pid=167}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732486585339.8ca644d2cada6dc44b363b7383dcd4fe. 2024-11-24T22:16:49,508 ERROR [RS_FLUSH_OPERATIONS-regionserver/bd6592c37bb2:0-2 {event_type=RS_FLUSH_REGIONS, pid=167}] handler.RSProcedureHandler(58): pid=167 java.io.IOException: Unable to complete flush {ENCODED => 8ca644d2cada6dc44b363b7383dcd4fe, NAME => 'TestAcidGuarantees,,1732486585339.8ca644d2cada6dc44b363b7383dcd4fe.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T22:16:49,508 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bd6592c37bb2:0-2 {event_type=RS_FLUSH_REGIONS, pid=167}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=167 java.io.IOException: Unable to complete flush {ENCODED => 8ca644d2cada6dc44b363b7383dcd4fe, NAME => 'TestAcidGuarantees,,1732486585339.8ca644d2cada6dc44b363b7383dcd4fe.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T22:16:49,508 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37701 {}] master.HMaster(4177): Remote procedure failed, pid=167 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 8ca644d2cada6dc44b363b7383dcd4fe, NAME => 'TestAcidGuarantees,,1732486585339.8ca644d2cada6dc44b363b7383dcd4fe.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 8ca644d2cada6dc44b363b7383dcd4fe, NAME => 'TestAcidGuarantees,,1732486585339.8ca644d2cada6dc44b363b7383dcd4fe.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T22:16:49,660 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=40087 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=167 2024-11-24T22:16:49,660 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bd6592c37bb2:0-0 {event_type=RS_FLUSH_REGIONS, pid=167}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732486585339.8ca644d2cada6dc44b363b7383dcd4fe. 2024-11-24T22:16:49,660 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bd6592c37bb2:0-0 {event_type=RS_FLUSH_REGIONS, pid=167}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732486585339.8ca644d2cada6dc44b363b7383dcd4fe. as already flushing 2024-11-24T22:16:49,660 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bd6592c37bb2:0-0 {event_type=RS_FLUSH_REGIONS, pid=167}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732486585339.8ca644d2cada6dc44b363b7383dcd4fe. 2024-11-24T22:16:49,660 ERROR [RS_FLUSH_OPERATIONS-regionserver/bd6592c37bb2:0-0 {event_type=RS_FLUSH_REGIONS, pid=167}] handler.RSProcedureHandler(58): pid=167 java.io.IOException: Unable to complete flush {ENCODED => 8ca644d2cada6dc44b363b7383dcd4fe, NAME => 'TestAcidGuarantees,,1732486585339.8ca644d2cada6dc44b363b7383dcd4fe.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T22:16:49,661 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bd6592c37bb2:0-0 {event_type=RS_FLUSH_REGIONS, pid=167}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=167 java.io.IOException: Unable to complete flush {ENCODED => 8ca644d2cada6dc44b363b7383dcd4fe, NAME => 'TestAcidGuarantees,,1732486585339.8ca644d2cada6dc44b363b7383dcd4fe.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T22:16:49,661 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37701 {}] master.HMaster(4177): Remote procedure failed, pid=167 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 8ca644d2cada6dc44b363b7383dcd4fe, NAME => 'TestAcidGuarantees,,1732486585339.8ca644d2cada6dc44b363b7383dcd4fe.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 8ca644d2cada6dc44b363b7383dcd4fe, NAME => 'TestAcidGuarantees,,1732486585339.8ca644d2cada6dc44b363b7383dcd4fe.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T22:16:49,670 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37701 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=166 2024-11-24T22:16:49,714 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=1005 (bloomFilter=true), to=hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/8ca644d2cada6dc44b363b7383dcd4fe/.tmp/C/90980d39589642b19b8884b4423eed49 2024-11-24T22:16:49,717 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/8ca644d2cada6dc44b363b7383dcd4fe/.tmp/A/3589d762efd14116bfc39bff83dbd9d9 as hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/8ca644d2cada6dc44b363b7383dcd4fe/A/3589d762efd14116bfc39bff83dbd9d9 2024-11-24T22:16:49,720 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/8ca644d2cada6dc44b363b7383dcd4fe/A/3589d762efd14116bfc39bff83dbd9d9, entries=200, sequenceid=1005, filesize=39.0 K 2024-11-24T22:16:49,720 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/8ca644d2cada6dc44b363b7383dcd4fe/.tmp/B/c6770c17f5d84d469ece27b38636bc53 as hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/8ca644d2cada6dc44b363b7383dcd4fe/B/c6770c17f5d84d469ece27b38636bc53 2024-11-24T22:16:49,723 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/8ca644d2cada6dc44b363b7383dcd4fe/B/c6770c17f5d84d469ece27b38636bc53, entries=150, sequenceid=1005, filesize=12.0 K 2024-11-24T22:16:49,723 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/8ca644d2cada6dc44b363b7383dcd4fe/.tmp/C/90980d39589642b19b8884b4423eed49 as hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/8ca644d2cada6dc44b363b7383dcd4fe/C/90980d39589642b19b8884b4423eed49 2024-11-24T22:16:49,725 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/8ca644d2cada6dc44b363b7383dcd4fe/C/90980d39589642b19b8884b4423eed49, entries=150, sequenceid=1005, filesize=12.0 K 2024-11-24T22:16:49,726 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~134.18 KB/137400, heapSize ~377.34 KB/386400, currentSize=46.96 KB/48090 for 8ca644d2cada6dc44b363b7383dcd4fe in 1653ms, sequenceid=1005, compaction requested=false 2024-11-24T22:16:49,726 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 8ca644d2cada6dc44b363b7383dcd4fe: 2024-11-24T22:16:49,813 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=40087 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=167 2024-11-24T22:16:49,813 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bd6592c37bb2:0-1 {event_type=RS_FLUSH_REGIONS, pid=167}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732486585339.8ca644d2cada6dc44b363b7383dcd4fe. 2024-11-24T22:16:49,813 INFO [RS_FLUSH_OPERATIONS-regionserver/bd6592c37bb2:0-1 {event_type=RS_FLUSH_REGIONS, pid=167}] regionserver.HRegion(2902): Flushing 8ca644d2cada6dc44b363b7383dcd4fe 3/3 column families, dataSize=46.96 KB heapSize=140.20 KB 2024-11-24T22:16:49,813 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bd6592c37bb2:0-1 {event_type=RS_FLUSH_REGIONS, pid=167}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 8ca644d2cada6dc44b363b7383dcd4fe, store=A 2024-11-24T22:16:49,813 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bd6592c37bb2:0-1 {event_type=RS_FLUSH_REGIONS, pid=167}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-24T22:16:49,813 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bd6592c37bb2:0-1 {event_type=RS_FLUSH_REGIONS, pid=167}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 8ca644d2cada6dc44b363b7383dcd4fe, store=B 2024-11-24T22:16:49,813 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bd6592c37bb2:0-1 {event_type=RS_FLUSH_REGIONS, pid=167}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-24T22:16:49,813 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bd6592c37bb2:0-1 {event_type=RS_FLUSH_REGIONS, pid=167}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 8ca644d2cada6dc44b363b7383dcd4fe, store=C 2024-11-24T22:16:49,813 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bd6592c37bb2:0-1 {event_type=RS_FLUSH_REGIONS, pid=167}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-24T22:16:49,817 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bd6592c37bb2:0-1 {event_type=RS_FLUSH_REGIONS, pid=167}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411243ea02dbc227041e8a709a83b3533db7a_8ca644d2cada6dc44b363b7383dcd4fe is 50, key is test_row_0/A:col10/1732486608075/Put/seqid=0 2024-11-24T22:16:49,820 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45929 is added to blk_1073742486_1662 (size=12454) 2024-11-24T22:16:50,221 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bd6592c37bb2:0-1 {event_type=RS_FLUSH_REGIONS, pid=167}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T22:16:50,223 INFO [RS_FLUSH_OPERATIONS-regionserver/bd6592c37bb2:0-1 {event_type=RS_FLUSH_REGIONS, pid=167}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411243ea02dbc227041e8a709a83b3533db7a_8ca644d2cada6dc44b363b7383dcd4fe to hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411243ea02dbc227041e8a709a83b3533db7a_8ca644d2cada6dc44b363b7383dcd4fe 2024-11-24T22:16:50,224 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bd6592c37bb2:0-1 {event_type=RS_FLUSH_REGIONS, pid=167}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/8ca644d2cada6dc44b363b7383dcd4fe/.tmp/A/0d373842ebe64020a36578cdb9bbe1bf, store: [table=TestAcidGuarantees family=A region=8ca644d2cada6dc44b363b7383dcd4fe] 2024-11-24T22:16:50,224 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bd6592c37bb2:0-1 {event_type=RS_FLUSH_REGIONS, pid=167}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/8ca644d2cada6dc44b363b7383dcd4fe/.tmp/A/0d373842ebe64020a36578cdb9bbe1bf is 175, key is test_row_0/A:col10/1732486608075/Put/seqid=0 2024-11-24T22:16:50,227 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45929 is added to blk_1073742487_1663 (size=31255) 2024-11-24T22:16:50,243 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40087 {}] regionserver.HRegion(8855): Flush requested on 8ca644d2cada6dc44b363b7383dcd4fe 2024-11-24T22:16:50,243 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732486585339.8ca644d2cada6dc44b363b7383dcd4fe. as already flushing 2024-11-24T22:16:50,243 INFO [Thread-2496 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-2496. 2024-11-24T22:16:50,243 DEBUG [Thread-2496 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-24T22:16:50,243 INFO [Thread-2498 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-2498. 2024-11-24T22:16:50,243 DEBUG [Thread-2496 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-24T22:16:50,243 DEBUG [Thread-2498 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-24T22:16:50,243 INFO [Thread-2502 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-2502. 2024-11-24T22:16:50,243 DEBUG [Thread-2498 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-24T22:16:50,243 DEBUG [Thread-2502 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-24T22:16:50,243 DEBUG [Thread-2502 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-24T22:16:50,243 DEBUG [Thread-2496 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-24T22:16:50,243 DEBUG [Thread-2498 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-24T22:16:50,243 DEBUG [Thread-2502 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-24T22:16:50,244 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-24T22:16:50,244 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-24T22:16:50,244 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-24T22:16:50,245 INFO [Thread-2500 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-2500. 2024-11-24T22:16:50,245 DEBUG [Thread-2500 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-24T22:16:50,245 INFO [Thread-2494 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-2494. 2024-11-24T22:16:50,245 DEBUG [Thread-2500 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-24T22:16:50,245 DEBUG [Thread-2494 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-24T22:16:50,245 DEBUG [Thread-2494 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-24T22:16:50,245 DEBUG [Thread-2500 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-24T22:16:50,245 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-24T22:16:50,245 DEBUG [Thread-2494 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-24T22:16:50,246 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-24T22:16:50,628 INFO [RS_FLUSH_OPERATIONS-regionserver/bd6592c37bb2:0-1 {event_type=RS_FLUSH_REGIONS, pid=167}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=1018, memsize=15.7 K, hasBloomFilter=true, into tmp file hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/8ca644d2cada6dc44b363b7383dcd4fe/.tmp/A/0d373842ebe64020a36578cdb9bbe1bf 2024-11-24T22:16:50,633 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bd6592c37bb2:0-1 {event_type=RS_FLUSH_REGIONS, pid=167}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/8ca644d2cada6dc44b363b7383dcd4fe/.tmp/B/9986550607554199af403f20b3ac0c89 is 50, key is test_row_0/B:col10/1732486608075/Put/seqid=0 2024-11-24T22:16:50,636 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45929 is added to blk_1073742488_1664 (size=12301) 2024-11-24T22:16:50,637 INFO [RS_FLUSH_OPERATIONS-regionserver/bd6592c37bb2:0-1 {event_type=RS_FLUSH_REGIONS, pid=167}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=1018 (bloomFilter=true), to=hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/8ca644d2cada6dc44b363b7383dcd4fe/.tmp/B/9986550607554199af403f20b3ac0c89 2024-11-24T22:16:50,641 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bd6592c37bb2:0-1 {event_type=RS_FLUSH_REGIONS, pid=167}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/8ca644d2cada6dc44b363b7383dcd4fe/.tmp/C/a020e316909c4bb7955baa0097597ff8 is 50, key is test_row_0/C:col10/1732486608075/Put/seqid=0 2024-11-24T22:16:50,643 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45929 is added to blk_1073742489_1665 (size=12301) 2024-11-24T22:16:51,044 INFO [RS_FLUSH_OPERATIONS-regionserver/bd6592c37bb2:0-1 {event_type=RS_FLUSH_REGIONS, pid=167}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=1018 (bloomFilter=true), to=hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/8ca644d2cada6dc44b363b7383dcd4fe/.tmp/C/a020e316909c4bb7955baa0097597ff8 2024-11-24T22:16:51,047 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bd6592c37bb2:0-1 {event_type=RS_FLUSH_REGIONS, pid=167}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/8ca644d2cada6dc44b363b7383dcd4fe/.tmp/A/0d373842ebe64020a36578cdb9bbe1bf as hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/8ca644d2cada6dc44b363b7383dcd4fe/A/0d373842ebe64020a36578cdb9bbe1bf 2024-11-24T22:16:51,049 INFO [RS_FLUSH_OPERATIONS-regionserver/bd6592c37bb2:0-1 {event_type=RS_FLUSH_REGIONS, pid=167}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/8ca644d2cada6dc44b363b7383dcd4fe/A/0d373842ebe64020a36578cdb9bbe1bf, entries=150, sequenceid=1018, filesize=30.5 K 2024-11-24T22:16:51,049 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bd6592c37bb2:0-1 {event_type=RS_FLUSH_REGIONS, pid=167}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/8ca644d2cada6dc44b363b7383dcd4fe/.tmp/B/9986550607554199af403f20b3ac0c89 as hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/8ca644d2cada6dc44b363b7383dcd4fe/B/9986550607554199af403f20b3ac0c89 2024-11-24T22:16:51,051 INFO [RS_FLUSH_OPERATIONS-regionserver/bd6592c37bb2:0-1 {event_type=RS_FLUSH_REGIONS, pid=167}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/8ca644d2cada6dc44b363b7383dcd4fe/B/9986550607554199af403f20b3ac0c89, entries=150, sequenceid=1018, filesize=12.0 K 2024-11-24T22:16:51,052 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bd6592c37bb2:0-1 {event_type=RS_FLUSH_REGIONS, pid=167}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/8ca644d2cada6dc44b363b7383dcd4fe/.tmp/C/a020e316909c4bb7955baa0097597ff8 as hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/8ca644d2cada6dc44b363b7383dcd4fe/C/a020e316909c4bb7955baa0097597ff8 2024-11-24T22:16:51,054 INFO [RS_FLUSH_OPERATIONS-regionserver/bd6592c37bb2:0-1 {event_type=RS_FLUSH_REGIONS, pid=167}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/8ca644d2cada6dc44b363b7383dcd4fe/C/a020e316909c4bb7955baa0097597ff8, entries=150, sequenceid=1018, filesize=12.0 K 2024-11-24T22:16:51,055 INFO [RS_FLUSH_OPERATIONS-regionserver/bd6592c37bb2:0-1 {event_type=RS_FLUSH_REGIONS, pid=167}] regionserver.HRegion(3140): Finished flush of dataSize ~46.96 KB/48090, heapSize ~140.16 KB/143520, currentSize=33.54 KB/34350 for 8ca644d2cada6dc44b363b7383dcd4fe in 1242ms, sequenceid=1018, compaction requested=true 2024-11-24T22:16:51,055 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bd6592c37bb2:0-1 {event_type=RS_FLUSH_REGIONS, pid=167}] regionserver.HRegion(2603): Flush status journal for 8ca644d2cada6dc44b363b7383dcd4fe: 2024-11-24T22:16:51,055 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bd6592c37bb2:0-1 {event_type=RS_FLUSH_REGIONS, pid=167}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732486585339.8ca644d2cada6dc44b363b7383dcd4fe. 2024-11-24T22:16:51,055 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bd6592c37bb2:0-1 {event_type=RS_FLUSH_REGIONS, pid=167}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=167 2024-11-24T22:16:51,055 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37701 {}] master.HMaster(4169): Remote procedure done, pid=167 2024-11-24T22:16:51,057 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=167, resume processing ppid=166 2024-11-24T22:16:51,057 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=167, ppid=166, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 3.5320 sec 2024-11-24T22:16:51,058 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=166, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=166, table=TestAcidGuarantees in 3.5360 sec 2024-11-24T22:16:51,691 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37701 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=166 2024-11-24T22:16:51,692 INFO [RPCClient-NioEventLoopGroup-4-12 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-24T22:16:51,692 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(392): Finished test. Writers: 2024-11-24T22:16:51,692 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 109 2024-11-24T22:16:51,692 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 237 2024-11-24T22:16:51,692 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 190 2024-11-24T22:16:51,692 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 173 2024-11-24T22:16:51,692 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 232 2024-11-24T22:16:51,692 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(396): Readers: 2024-11-24T22:16:51,692 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(398): read 2670 2024-11-24T22:16:51,692 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(398): read 2610 2024-11-24T22:16:51,692 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(398): read 2604 2024-11-24T22:16:51,692 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(398): read 2678 2024-11-24T22:16:51,692 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(398): read 2624 2024-11-24T22:16:51,692 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(400): Scanners: 2024-11-24T22:16:51,692 INFO [Time-limited test {}] client.AsyncConnectionImpl(233): Connection has been closed by Time-limited test. 2024-11-24T22:16:51,692 DEBUG [Time-limited test {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool.doWork(AcidGuaranteesTestTool.java:131) at org.apache.hadoop.hbase.util.AbstractHBaseTool.run(AbstractHBaseTool.java:150) at org.apache.hadoop.hbase.AcidGuaranteesTestBase.runTestAtomicity(AcidGuaranteesTestBase.java:102) at org.apache.hadoop.hbase.AcidGuaranteesTestBase.testMobGetAtomicity(AcidGuaranteesTestBase.java:122) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) at java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) at java.base/java.lang.reflect.Method.invoke(Method.java:568) at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) at java.base/java.util.concurrent.FutureTask.run(FutureTask.java:264) at java.base/java.lang.Thread.run(Thread.java:840) 2024-11-24T22:16:51,692 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-24T22:16:51,692 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-24T22:16:51,692 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-24T22:16:51,693 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37701 {}] master.HMaster$13(2820): Client=jenkins//172.17.0.2 disable TestAcidGuarantees 2024-11-24T22:16:51,694 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37701 {}] procedure2.ProcedureExecutor(1139): Stored pid=168, state=RUNNABLE:DISABLE_TABLE_PREPARE, hasLock=false; DisableTableProcedure table=TestAcidGuarantees 2024-11-24T22:16:51,695 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37701 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=168 2024-11-24T22:16:51,695 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1732486611695"}]},"ts":"1732486611695"} 2024-11-24T22:16:51,696 INFO [PEWorker-1 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=DISABLING in hbase:meta 2024-11-24T22:16:51,696 INFO [PEWorker-1 {}] procedure.DisableTableProcedure(284): Set TestAcidGuarantees to state=DISABLING 2024-11-24T22:16:51,697 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=169, ppid=168, state=RUNNABLE:CLOSE_TABLE_REGIONS_SCHEDULE, hasLock=false; CloseTableRegionsProcedure table=TestAcidGuarantees}] 2024-11-24T22:16:51,698 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=170, ppid=169, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=8ca644d2cada6dc44b363b7383dcd4fe, UNASSIGN}] 2024-11-24T22:16:51,699 INFO [PEWorker-4 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=170, ppid=169, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=8ca644d2cada6dc44b363b7383dcd4fe, UNASSIGN 2024-11-24T22:16:51,699 INFO [PEWorker-4 {}] assignment.RegionStateStore(223): pid=170 updating hbase:meta row=8ca644d2cada6dc44b363b7383dcd4fe, regionState=CLOSING, regionLocation=bd6592c37bb2,40087,1732486423713 2024-11-24T22:16:51,700 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=170, ppid=169, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=8ca644d2cada6dc44b363b7383dcd4fe, UNASSIGN because future has completed 2024-11-24T22:16:51,700 DEBUG [PEWorker-5 {}] assignment.TransitRegionStateProcedure(375): Close region: isSplit: false: evictOnSplit: true: evictOnClose: false 2024-11-24T22:16:51,700 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=171, ppid=170, state=RUNNABLE, hasLock=false; CloseRegionProcedure 8ca644d2cada6dc44b363b7383dcd4fe, server=bd6592c37bb2,40087,1732486423713}] 2024-11-24T22:16:51,801 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37701 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=168 2024-11-24T22:16:51,852 INFO [RS_CLOSE_REGION-regionserver/bd6592c37bb2:0-0 {event_type=M_RS_CLOSE_REGION, pid=171}] handler.UnassignRegionHandler(122): Close 8ca644d2cada6dc44b363b7383dcd4fe 2024-11-24T22:16:51,852 DEBUG [RS_CLOSE_REGION-regionserver/bd6592c37bb2:0-0 {event_type=M_RS_CLOSE_REGION, pid=171}] handler.UnassignRegionHandler(136): Unassign region: split region: false: evictCache: false 2024-11-24T22:16:51,852 DEBUG [RS_CLOSE_REGION-regionserver/bd6592c37bb2:0-0 {event_type=M_RS_CLOSE_REGION, pid=171}] regionserver.HRegion(1722): Closing 8ca644d2cada6dc44b363b7383dcd4fe, disabling compactions & flushes 2024-11-24T22:16:51,852 INFO [RS_CLOSE_REGION-regionserver/bd6592c37bb2:0-0 {event_type=M_RS_CLOSE_REGION, pid=171}] regionserver.HRegion(1755): Closing region TestAcidGuarantees,,1732486585339.8ca644d2cada6dc44b363b7383dcd4fe. 2024-11-24T22:16:51,852 DEBUG [RS_CLOSE_REGION-regionserver/bd6592c37bb2:0-0 {event_type=M_RS_CLOSE_REGION, pid=171}] regionserver.HRegion(1776): Time limited wait for close lock on TestAcidGuarantees,,1732486585339.8ca644d2cada6dc44b363b7383dcd4fe. 2024-11-24T22:16:51,852 DEBUG [RS_CLOSE_REGION-regionserver/bd6592c37bb2:0-0 {event_type=M_RS_CLOSE_REGION, pid=171}] regionserver.HRegion(1843): Acquired close lock on TestAcidGuarantees,,1732486585339.8ca644d2cada6dc44b363b7383dcd4fe. after waiting 0 ms 2024-11-24T22:16:51,852 DEBUG [RS_CLOSE_REGION-regionserver/bd6592c37bb2:0-0 {event_type=M_RS_CLOSE_REGION, pid=171}] regionserver.HRegion(1853): Updates disabled for region TestAcidGuarantees,,1732486585339.8ca644d2cada6dc44b363b7383dcd4fe. 2024-11-24T22:16:51,852 INFO [RS_CLOSE_REGION-regionserver/bd6592c37bb2:0-0 {event_type=M_RS_CLOSE_REGION, pid=171}] regionserver.HRegion(2902): Flushing 8ca644d2cada6dc44b363b7383dcd4fe 3/3 column families, dataSize=33.54 KB heapSize=100.36 KB 2024-11-24T22:16:51,853 DEBUG [RS_CLOSE_REGION-regionserver/bd6592c37bb2:0-0 {event_type=M_RS_CLOSE_REGION, pid=171}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 8ca644d2cada6dc44b363b7383dcd4fe, store=A 2024-11-24T22:16:51,853 DEBUG [RS_CLOSE_REGION-regionserver/bd6592c37bb2:0-0 {event_type=M_RS_CLOSE_REGION, pid=171}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-24T22:16:51,853 DEBUG [RS_CLOSE_REGION-regionserver/bd6592c37bb2:0-0 {event_type=M_RS_CLOSE_REGION, pid=171}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 8ca644d2cada6dc44b363b7383dcd4fe, store=B 2024-11-24T22:16:51,853 DEBUG [RS_CLOSE_REGION-regionserver/bd6592c37bb2:0-0 {event_type=M_RS_CLOSE_REGION, pid=171}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-24T22:16:51,853 DEBUG [RS_CLOSE_REGION-regionserver/bd6592c37bb2:0-0 {event_type=M_RS_CLOSE_REGION, pid=171}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 8ca644d2cada6dc44b363b7383dcd4fe, store=C 2024-11-24T22:16:51,853 DEBUG [RS_CLOSE_REGION-regionserver/bd6592c37bb2:0-0 {event_type=M_RS_CLOSE_REGION, pid=171}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-24T22:16:51,857 DEBUG [RS_CLOSE_REGION-regionserver/bd6592c37bb2:0-0 {event_type=M_RS_CLOSE_REGION, pid=171}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241124c462436d746b4a35bcad74286527b5c6_8ca644d2cada6dc44b363b7383dcd4fe is 50, key is test_row_0/A:col10/1732486610243/Put/seqid=0 2024-11-24T22:16:51,860 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45929 is added to blk_1073742490_1666 (size=9914) 2024-11-24T22:16:52,011 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37701 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=168 2024-11-24T22:16:52,260 DEBUG [RS_CLOSE_REGION-regionserver/bd6592c37bb2:0-0 {event_type=M_RS_CLOSE_REGION, pid=171}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T22:16:52,263 INFO [RS_CLOSE_REGION-regionserver/bd6592c37bb2:0-0 {event_type=M_RS_CLOSE_REGION, pid=171}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241124c462436d746b4a35bcad74286527b5c6_8ca644d2cada6dc44b363b7383dcd4fe to hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241124c462436d746b4a35bcad74286527b5c6_8ca644d2cada6dc44b363b7383dcd4fe 2024-11-24T22:16:52,264 DEBUG [RS_CLOSE_REGION-regionserver/bd6592c37bb2:0-0 {event_type=M_RS_CLOSE_REGION, pid=171}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/8ca644d2cada6dc44b363b7383dcd4fe/.tmp/A/f30a3ace6b014020ac8f8da210794c6c, store: [table=TestAcidGuarantees family=A region=8ca644d2cada6dc44b363b7383dcd4fe] 2024-11-24T22:16:52,264 DEBUG [RS_CLOSE_REGION-regionserver/bd6592c37bb2:0-0 {event_type=M_RS_CLOSE_REGION, pid=171}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/8ca644d2cada6dc44b363b7383dcd4fe/.tmp/A/f30a3ace6b014020ac8f8da210794c6c is 175, key is test_row_0/A:col10/1732486610243/Put/seqid=0 2024-11-24T22:16:52,267 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45929 is added to blk_1073742491_1667 (size=22561) 2024-11-24T22:16:52,321 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37701 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=168 2024-11-24T22:16:52,668 INFO [RS_CLOSE_REGION-regionserver/bd6592c37bb2:0-0 {event_type=M_RS_CLOSE_REGION, pid=171}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=1026, memsize=11.2 K, hasBloomFilter=true, into tmp file hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/8ca644d2cada6dc44b363b7383dcd4fe/.tmp/A/f30a3ace6b014020ac8f8da210794c6c 2024-11-24T22:16:52,673 DEBUG [RS_CLOSE_REGION-regionserver/bd6592c37bb2:0-0 {event_type=M_RS_CLOSE_REGION, pid=171}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/8ca644d2cada6dc44b363b7383dcd4fe/.tmp/B/ceca785ebc18479b946c77f66d69cb33 is 50, key is test_row_0/B:col10/1732486610243/Put/seqid=0 2024-11-24T22:16:52,676 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45929 is added to blk_1073742492_1668 (size=9857) 2024-11-24T22:16:52,831 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37701 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=168 2024-11-24T22:16:53,076 INFO [RS_CLOSE_REGION-regionserver/bd6592c37bb2:0-0 {event_type=M_RS_CLOSE_REGION, pid=171}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=11.18 KB at sequenceid=1026 (bloomFilter=true), to=hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/8ca644d2cada6dc44b363b7383dcd4fe/.tmp/B/ceca785ebc18479b946c77f66d69cb33 2024-11-24T22:16:53,081 DEBUG [RS_CLOSE_REGION-regionserver/bd6592c37bb2:0-0 {event_type=M_RS_CLOSE_REGION, pid=171}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/8ca644d2cada6dc44b363b7383dcd4fe/.tmp/C/9b5141abb9cf461b843e321b6ea342e9 is 50, key is test_row_0/C:col10/1732486610243/Put/seqid=0 2024-11-24T22:16:53,083 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45929 is added to blk_1073742493_1669 (size=9857) 2024-11-24T22:16:53,484 INFO [RS_CLOSE_REGION-regionserver/bd6592c37bb2:0-0 {event_type=M_RS_CLOSE_REGION, pid=171}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=11.18 KB at sequenceid=1026 (bloomFilter=true), to=hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/8ca644d2cada6dc44b363b7383dcd4fe/.tmp/C/9b5141abb9cf461b843e321b6ea342e9 2024-11-24T22:16:53,487 DEBUG [RS_CLOSE_REGION-regionserver/bd6592c37bb2:0-0 {event_type=M_RS_CLOSE_REGION, pid=171}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/8ca644d2cada6dc44b363b7383dcd4fe/.tmp/A/f30a3ace6b014020ac8f8da210794c6c as hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/8ca644d2cada6dc44b363b7383dcd4fe/A/f30a3ace6b014020ac8f8da210794c6c 2024-11-24T22:16:53,490 INFO [RS_CLOSE_REGION-regionserver/bd6592c37bb2:0-0 {event_type=M_RS_CLOSE_REGION, pid=171}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/8ca644d2cada6dc44b363b7383dcd4fe/A/f30a3ace6b014020ac8f8da210794c6c, entries=100, sequenceid=1026, filesize=22.0 K 2024-11-24T22:16:53,490 DEBUG [RS_CLOSE_REGION-regionserver/bd6592c37bb2:0-0 {event_type=M_RS_CLOSE_REGION, pid=171}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/8ca644d2cada6dc44b363b7383dcd4fe/.tmp/B/ceca785ebc18479b946c77f66d69cb33 as hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/8ca644d2cada6dc44b363b7383dcd4fe/B/ceca785ebc18479b946c77f66d69cb33 2024-11-24T22:16:53,492 INFO [RS_CLOSE_REGION-regionserver/bd6592c37bb2:0-0 {event_type=M_RS_CLOSE_REGION, pid=171}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/8ca644d2cada6dc44b363b7383dcd4fe/B/ceca785ebc18479b946c77f66d69cb33, entries=100, sequenceid=1026, filesize=9.6 K 2024-11-24T22:16:53,493 DEBUG [RS_CLOSE_REGION-regionserver/bd6592c37bb2:0-0 {event_type=M_RS_CLOSE_REGION, pid=171}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/8ca644d2cada6dc44b363b7383dcd4fe/.tmp/C/9b5141abb9cf461b843e321b6ea342e9 as hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/8ca644d2cada6dc44b363b7383dcd4fe/C/9b5141abb9cf461b843e321b6ea342e9 2024-11-24T22:16:53,495 INFO [RS_CLOSE_REGION-regionserver/bd6592c37bb2:0-0 {event_type=M_RS_CLOSE_REGION, pid=171}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/8ca644d2cada6dc44b363b7383dcd4fe/C/9b5141abb9cf461b843e321b6ea342e9, entries=100, sequenceid=1026, filesize=9.6 K 2024-11-24T22:16:53,496 INFO [RS_CLOSE_REGION-regionserver/bd6592c37bb2:0-0 {event_type=M_RS_CLOSE_REGION, pid=171}] regionserver.HRegion(3140): Finished flush of dataSize ~33.54 KB/34350, heapSize ~100.31 KB/102720, currentSize=0 B/0 for 8ca644d2cada6dc44b363b7383dcd4fe in 1644ms, sequenceid=1026, compaction requested=true 2024-11-24T22:16:53,496 DEBUG [StoreCloser-TestAcidGuarantees,,1732486585339.8ca644d2cada6dc44b363b7383dcd4fe.-1 {}] regionserver.HStore(2317): Moving the files [hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/8ca644d2cada6dc44b363b7383dcd4fe/A/4d8ef082bbc741109521c9a43715f049, hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/8ca644d2cada6dc44b363b7383dcd4fe/A/35a85100955447888670863594d84360, hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/8ca644d2cada6dc44b363b7383dcd4fe/A/ff60faeb807c4decab1b2a1f7ba31d85, hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/8ca644d2cada6dc44b363b7383dcd4fe/A/ba62e5cf94804332b0c6667d4633e13d, hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/8ca644d2cada6dc44b363b7383dcd4fe/A/fceb773ae4244af2814973d66416d959, hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/8ca644d2cada6dc44b363b7383dcd4fe/A/c30dfe64f4b24555a71a47fb4cd35e11, hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/8ca644d2cada6dc44b363b7383dcd4fe/A/3cadf84e7037408481c122039cbf1085, hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/8ca644d2cada6dc44b363b7383dcd4fe/A/4c11b34b2e20404193582684f79f36df, hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/8ca644d2cada6dc44b363b7383dcd4fe/A/7b016023da544892afa2a8f1390bb388, hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/8ca644d2cada6dc44b363b7383dcd4fe/A/b722343fe5fc48559de2c4b094712e51, hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/8ca644d2cada6dc44b363b7383dcd4fe/A/c679a2427e9f4fcea97a02498a755dde, hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/8ca644d2cada6dc44b363b7383dcd4fe/A/dd9c9897b4964ee9af1242e61e3c3d45, hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/8ca644d2cada6dc44b363b7383dcd4fe/A/09f6f98a4dde46df9e1923bd989b27de, hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/8ca644d2cada6dc44b363b7383dcd4fe/A/b8d8d281b8a04eeb8945f0979f39bfcc, hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/8ca644d2cada6dc44b363b7383dcd4fe/A/7263a115caa04d7d9b7c84d0a6c9cce4, hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/8ca644d2cada6dc44b363b7383dcd4fe/A/53ee59bfb6b54d9f8799663736d3c46a, hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/8ca644d2cada6dc44b363b7383dcd4fe/A/99464ebe699642cfa110755eb79a0020, hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/8ca644d2cada6dc44b363b7383dcd4fe/A/c18f1dc8317f4922af06f6e37bc834ac, hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/8ca644d2cada6dc44b363b7383dcd4fe/A/18acc92875294311b85e9cf8ff590296, hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/8ca644d2cada6dc44b363b7383dcd4fe/A/aca73ecfca3c475c865c0a6c969e035c, hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/8ca644d2cada6dc44b363b7383dcd4fe/A/5b75c2f936b144c4a5e4bf896c7a3178, hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/8ca644d2cada6dc44b363b7383dcd4fe/A/edcdc9c2d1544300b712df3846855e54, hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/8ca644d2cada6dc44b363b7383dcd4fe/A/596287ff0e7a4c05939060cb18065271] to archive 2024-11-24T22:16:53,497 DEBUG [StoreCloser-TestAcidGuarantees,,1732486585339.8ca644d2cada6dc44b363b7383dcd4fe.-1 {}] backup.HFileArchiver(360): Archiving compacted files. 2024-11-24T22:16:53,498 DEBUG [StoreCloser-TestAcidGuarantees,,1732486585339.8ca644d2cada6dc44b363b7383dcd4fe.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/8ca644d2cada6dc44b363b7383dcd4fe/A/4d8ef082bbc741109521c9a43715f049 to hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/archive/data/default/TestAcidGuarantees/8ca644d2cada6dc44b363b7383dcd4fe/A/4d8ef082bbc741109521c9a43715f049 2024-11-24T22:16:53,499 DEBUG [StoreCloser-TestAcidGuarantees,,1732486585339.8ca644d2cada6dc44b363b7383dcd4fe.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/8ca644d2cada6dc44b363b7383dcd4fe/A/35a85100955447888670863594d84360 to hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/archive/data/default/TestAcidGuarantees/8ca644d2cada6dc44b363b7383dcd4fe/A/35a85100955447888670863594d84360 2024-11-24T22:16:53,499 DEBUG [StoreCloser-TestAcidGuarantees,,1732486585339.8ca644d2cada6dc44b363b7383dcd4fe.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/8ca644d2cada6dc44b363b7383dcd4fe/A/ff60faeb807c4decab1b2a1f7ba31d85 to hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/archive/data/default/TestAcidGuarantees/8ca644d2cada6dc44b363b7383dcd4fe/A/ff60faeb807c4decab1b2a1f7ba31d85 2024-11-24T22:16:53,500 DEBUG [StoreCloser-TestAcidGuarantees,,1732486585339.8ca644d2cada6dc44b363b7383dcd4fe.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/8ca644d2cada6dc44b363b7383dcd4fe/A/ba62e5cf94804332b0c6667d4633e13d to hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/archive/data/default/TestAcidGuarantees/8ca644d2cada6dc44b363b7383dcd4fe/A/ba62e5cf94804332b0c6667d4633e13d 2024-11-24T22:16:53,501 DEBUG [StoreCloser-TestAcidGuarantees,,1732486585339.8ca644d2cada6dc44b363b7383dcd4fe.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/8ca644d2cada6dc44b363b7383dcd4fe/A/fceb773ae4244af2814973d66416d959 to hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/archive/data/default/TestAcidGuarantees/8ca644d2cada6dc44b363b7383dcd4fe/A/fceb773ae4244af2814973d66416d959 2024-11-24T22:16:53,502 DEBUG [StoreCloser-TestAcidGuarantees,,1732486585339.8ca644d2cada6dc44b363b7383dcd4fe.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/8ca644d2cada6dc44b363b7383dcd4fe/A/c30dfe64f4b24555a71a47fb4cd35e11 to hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/archive/data/default/TestAcidGuarantees/8ca644d2cada6dc44b363b7383dcd4fe/A/c30dfe64f4b24555a71a47fb4cd35e11 2024-11-24T22:16:53,503 DEBUG [StoreCloser-TestAcidGuarantees,,1732486585339.8ca644d2cada6dc44b363b7383dcd4fe.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/8ca644d2cada6dc44b363b7383dcd4fe/A/3cadf84e7037408481c122039cbf1085 to hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/archive/data/default/TestAcidGuarantees/8ca644d2cada6dc44b363b7383dcd4fe/A/3cadf84e7037408481c122039cbf1085 2024-11-24T22:16:53,503 DEBUG [StoreCloser-TestAcidGuarantees,,1732486585339.8ca644d2cada6dc44b363b7383dcd4fe.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/8ca644d2cada6dc44b363b7383dcd4fe/A/4c11b34b2e20404193582684f79f36df to hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/archive/data/default/TestAcidGuarantees/8ca644d2cada6dc44b363b7383dcd4fe/A/4c11b34b2e20404193582684f79f36df 2024-11-24T22:16:53,504 DEBUG [StoreCloser-TestAcidGuarantees,,1732486585339.8ca644d2cada6dc44b363b7383dcd4fe.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/8ca644d2cada6dc44b363b7383dcd4fe/A/7b016023da544892afa2a8f1390bb388 to hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/archive/data/default/TestAcidGuarantees/8ca644d2cada6dc44b363b7383dcd4fe/A/7b016023da544892afa2a8f1390bb388 2024-11-24T22:16:53,505 DEBUG [StoreCloser-TestAcidGuarantees,,1732486585339.8ca644d2cada6dc44b363b7383dcd4fe.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/8ca644d2cada6dc44b363b7383dcd4fe/A/b722343fe5fc48559de2c4b094712e51 to hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/archive/data/default/TestAcidGuarantees/8ca644d2cada6dc44b363b7383dcd4fe/A/b722343fe5fc48559de2c4b094712e51 2024-11-24T22:16:53,505 DEBUG [StoreCloser-TestAcidGuarantees,,1732486585339.8ca644d2cada6dc44b363b7383dcd4fe.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/8ca644d2cada6dc44b363b7383dcd4fe/A/c679a2427e9f4fcea97a02498a755dde to hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/archive/data/default/TestAcidGuarantees/8ca644d2cada6dc44b363b7383dcd4fe/A/c679a2427e9f4fcea97a02498a755dde 2024-11-24T22:16:53,506 DEBUG [StoreCloser-TestAcidGuarantees,,1732486585339.8ca644d2cada6dc44b363b7383dcd4fe.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/8ca644d2cada6dc44b363b7383dcd4fe/A/dd9c9897b4964ee9af1242e61e3c3d45 to hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/archive/data/default/TestAcidGuarantees/8ca644d2cada6dc44b363b7383dcd4fe/A/dd9c9897b4964ee9af1242e61e3c3d45 2024-11-24T22:16:53,507 DEBUG [StoreCloser-TestAcidGuarantees,,1732486585339.8ca644d2cada6dc44b363b7383dcd4fe.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/8ca644d2cada6dc44b363b7383dcd4fe/A/09f6f98a4dde46df9e1923bd989b27de to hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/archive/data/default/TestAcidGuarantees/8ca644d2cada6dc44b363b7383dcd4fe/A/09f6f98a4dde46df9e1923bd989b27de 2024-11-24T22:16:53,507 DEBUG [StoreCloser-TestAcidGuarantees,,1732486585339.8ca644d2cada6dc44b363b7383dcd4fe.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/8ca644d2cada6dc44b363b7383dcd4fe/A/b8d8d281b8a04eeb8945f0979f39bfcc to hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/archive/data/default/TestAcidGuarantees/8ca644d2cada6dc44b363b7383dcd4fe/A/b8d8d281b8a04eeb8945f0979f39bfcc 2024-11-24T22:16:53,508 DEBUG [StoreCloser-TestAcidGuarantees,,1732486585339.8ca644d2cada6dc44b363b7383dcd4fe.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/8ca644d2cada6dc44b363b7383dcd4fe/A/7263a115caa04d7d9b7c84d0a6c9cce4 to hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/archive/data/default/TestAcidGuarantees/8ca644d2cada6dc44b363b7383dcd4fe/A/7263a115caa04d7d9b7c84d0a6c9cce4 2024-11-24T22:16:53,509 DEBUG [StoreCloser-TestAcidGuarantees,,1732486585339.8ca644d2cada6dc44b363b7383dcd4fe.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/8ca644d2cada6dc44b363b7383dcd4fe/A/53ee59bfb6b54d9f8799663736d3c46a to hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/archive/data/default/TestAcidGuarantees/8ca644d2cada6dc44b363b7383dcd4fe/A/53ee59bfb6b54d9f8799663736d3c46a 2024-11-24T22:16:53,509 DEBUG [StoreCloser-TestAcidGuarantees,,1732486585339.8ca644d2cada6dc44b363b7383dcd4fe.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/8ca644d2cada6dc44b363b7383dcd4fe/A/99464ebe699642cfa110755eb79a0020 to hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/archive/data/default/TestAcidGuarantees/8ca644d2cada6dc44b363b7383dcd4fe/A/99464ebe699642cfa110755eb79a0020 2024-11-24T22:16:53,510 DEBUG [StoreCloser-TestAcidGuarantees,,1732486585339.8ca644d2cada6dc44b363b7383dcd4fe.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/8ca644d2cada6dc44b363b7383dcd4fe/A/c18f1dc8317f4922af06f6e37bc834ac to hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/archive/data/default/TestAcidGuarantees/8ca644d2cada6dc44b363b7383dcd4fe/A/c18f1dc8317f4922af06f6e37bc834ac 2024-11-24T22:16:53,511 DEBUG [StoreCloser-TestAcidGuarantees,,1732486585339.8ca644d2cada6dc44b363b7383dcd4fe.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/8ca644d2cada6dc44b363b7383dcd4fe/A/18acc92875294311b85e9cf8ff590296 to hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/archive/data/default/TestAcidGuarantees/8ca644d2cada6dc44b363b7383dcd4fe/A/18acc92875294311b85e9cf8ff590296 2024-11-24T22:16:53,511 DEBUG [StoreCloser-TestAcidGuarantees,,1732486585339.8ca644d2cada6dc44b363b7383dcd4fe.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/8ca644d2cada6dc44b363b7383dcd4fe/A/aca73ecfca3c475c865c0a6c969e035c to hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/archive/data/default/TestAcidGuarantees/8ca644d2cada6dc44b363b7383dcd4fe/A/aca73ecfca3c475c865c0a6c969e035c 2024-11-24T22:16:53,512 DEBUG [StoreCloser-TestAcidGuarantees,,1732486585339.8ca644d2cada6dc44b363b7383dcd4fe.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/8ca644d2cada6dc44b363b7383dcd4fe/A/5b75c2f936b144c4a5e4bf896c7a3178 to hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/archive/data/default/TestAcidGuarantees/8ca644d2cada6dc44b363b7383dcd4fe/A/5b75c2f936b144c4a5e4bf896c7a3178 2024-11-24T22:16:53,513 DEBUG [StoreCloser-TestAcidGuarantees,,1732486585339.8ca644d2cada6dc44b363b7383dcd4fe.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/8ca644d2cada6dc44b363b7383dcd4fe/A/edcdc9c2d1544300b712df3846855e54 to hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/archive/data/default/TestAcidGuarantees/8ca644d2cada6dc44b363b7383dcd4fe/A/edcdc9c2d1544300b712df3846855e54 2024-11-24T22:16:53,513 DEBUG [StoreCloser-TestAcidGuarantees,,1732486585339.8ca644d2cada6dc44b363b7383dcd4fe.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/8ca644d2cada6dc44b363b7383dcd4fe/A/596287ff0e7a4c05939060cb18065271 to hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/archive/data/default/TestAcidGuarantees/8ca644d2cada6dc44b363b7383dcd4fe/A/596287ff0e7a4c05939060cb18065271 2024-11-24T22:16:53,514 DEBUG [StoreCloser-TestAcidGuarantees,,1732486585339.8ca644d2cada6dc44b363b7383dcd4fe.-1 {}] regionserver.HStore(2317): Moving the files [hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/8ca644d2cada6dc44b363b7383dcd4fe/B/be99041ff522499f83a39a00787ed3ce, hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/8ca644d2cada6dc44b363b7383dcd4fe/B/c807ab7919a94e47a5f40b2f71d3a197, hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/8ca644d2cada6dc44b363b7383dcd4fe/B/9e0fd71ad3754b308cad5b3311354f00, hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/8ca644d2cada6dc44b363b7383dcd4fe/B/36dd327c79934afa8769997291f671da, hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/8ca644d2cada6dc44b363b7383dcd4fe/B/0872fb6cf5634f7c97a9662491517a97, hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/8ca644d2cada6dc44b363b7383dcd4fe/B/38029836f3024bea8777a5b68b4ad875, hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/8ca644d2cada6dc44b363b7383dcd4fe/B/38483da317664eb19a79deb055a938d4, hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/8ca644d2cada6dc44b363b7383dcd4fe/B/4a689bc4d8984c48b58ed987cdb599df, hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/8ca644d2cada6dc44b363b7383dcd4fe/B/5037fa9e4f23489aa5eede59338d5122, hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/8ca644d2cada6dc44b363b7383dcd4fe/B/f7f8edc5d3b149d5a11aa186b622f761, hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/8ca644d2cada6dc44b363b7383dcd4fe/B/31584a242fc64dfcb888a07f8cbddd9f, hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/8ca644d2cada6dc44b363b7383dcd4fe/B/730efb19372148f8bdb95e42b451eb85, hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/8ca644d2cada6dc44b363b7383dcd4fe/B/6e64a39e7a474ab9a6e384c6fbeb34d3, hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/8ca644d2cada6dc44b363b7383dcd4fe/B/0a9cb8108ab84a25a32039517a52cbeb, hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/8ca644d2cada6dc44b363b7383dcd4fe/B/dc23ac5e77d74dc0906774d8a726505d, hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/8ca644d2cada6dc44b363b7383dcd4fe/B/63e6053a21424b89a81ca99ff0a18861, hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/8ca644d2cada6dc44b363b7383dcd4fe/B/6b52569f4db642aebd605b9ea3871fa5, hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/8ca644d2cada6dc44b363b7383dcd4fe/B/e868df9854d3473fa99b4edb9eb7e4b9, hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/8ca644d2cada6dc44b363b7383dcd4fe/B/3b068b24fdb74c6190094f2727449194, hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/8ca644d2cada6dc44b363b7383dcd4fe/B/1eaf0718bad24a90ab90b86f3a35f33f, hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/8ca644d2cada6dc44b363b7383dcd4fe/B/60a721145bed4ac1ac6c4ee15134ccdb, hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/8ca644d2cada6dc44b363b7383dcd4fe/B/8b5d192284234dd797b73ec2a04209f8, hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/8ca644d2cada6dc44b363b7383dcd4fe/B/bfe3e61f56434ab781a4929b3389a3c6] to archive 2024-11-24T22:16:53,515 DEBUG [StoreCloser-TestAcidGuarantees,,1732486585339.8ca644d2cada6dc44b363b7383dcd4fe.-1 {}] backup.HFileArchiver(360): Archiving compacted files. 2024-11-24T22:16:53,516 DEBUG [StoreCloser-TestAcidGuarantees,,1732486585339.8ca644d2cada6dc44b363b7383dcd4fe.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/8ca644d2cada6dc44b363b7383dcd4fe/B/be99041ff522499f83a39a00787ed3ce to hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/archive/data/default/TestAcidGuarantees/8ca644d2cada6dc44b363b7383dcd4fe/B/be99041ff522499f83a39a00787ed3ce 2024-11-24T22:16:53,517 DEBUG [StoreCloser-TestAcidGuarantees,,1732486585339.8ca644d2cada6dc44b363b7383dcd4fe.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/8ca644d2cada6dc44b363b7383dcd4fe/B/c807ab7919a94e47a5f40b2f71d3a197 to hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/archive/data/default/TestAcidGuarantees/8ca644d2cada6dc44b363b7383dcd4fe/B/c807ab7919a94e47a5f40b2f71d3a197 2024-11-24T22:16:53,517 DEBUG [StoreCloser-TestAcidGuarantees,,1732486585339.8ca644d2cada6dc44b363b7383dcd4fe.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/8ca644d2cada6dc44b363b7383dcd4fe/B/9e0fd71ad3754b308cad5b3311354f00 to hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/archive/data/default/TestAcidGuarantees/8ca644d2cada6dc44b363b7383dcd4fe/B/9e0fd71ad3754b308cad5b3311354f00 2024-11-24T22:16:53,518 DEBUG [StoreCloser-TestAcidGuarantees,,1732486585339.8ca644d2cada6dc44b363b7383dcd4fe.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/8ca644d2cada6dc44b363b7383dcd4fe/B/36dd327c79934afa8769997291f671da to hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/archive/data/default/TestAcidGuarantees/8ca644d2cada6dc44b363b7383dcd4fe/B/36dd327c79934afa8769997291f671da 2024-11-24T22:16:53,519 DEBUG [StoreCloser-TestAcidGuarantees,,1732486585339.8ca644d2cada6dc44b363b7383dcd4fe.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/8ca644d2cada6dc44b363b7383dcd4fe/B/0872fb6cf5634f7c97a9662491517a97 to hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/archive/data/default/TestAcidGuarantees/8ca644d2cada6dc44b363b7383dcd4fe/B/0872fb6cf5634f7c97a9662491517a97 2024-11-24T22:16:53,519 DEBUG [StoreCloser-TestAcidGuarantees,,1732486585339.8ca644d2cada6dc44b363b7383dcd4fe.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/8ca644d2cada6dc44b363b7383dcd4fe/B/38029836f3024bea8777a5b68b4ad875 to hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/archive/data/default/TestAcidGuarantees/8ca644d2cada6dc44b363b7383dcd4fe/B/38029836f3024bea8777a5b68b4ad875 2024-11-24T22:16:53,520 DEBUG [StoreCloser-TestAcidGuarantees,,1732486585339.8ca644d2cada6dc44b363b7383dcd4fe.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/8ca644d2cada6dc44b363b7383dcd4fe/B/38483da317664eb19a79deb055a938d4 to hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/archive/data/default/TestAcidGuarantees/8ca644d2cada6dc44b363b7383dcd4fe/B/38483da317664eb19a79deb055a938d4 2024-11-24T22:16:53,521 DEBUG [StoreCloser-TestAcidGuarantees,,1732486585339.8ca644d2cada6dc44b363b7383dcd4fe.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/8ca644d2cada6dc44b363b7383dcd4fe/B/4a689bc4d8984c48b58ed987cdb599df to hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/archive/data/default/TestAcidGuarantees/8ca644d2cada6dc44b363b7383dcd4fe/B/4a689bc4d8984c48b58ed987cdb599df 2024-11-24T22:16:53,521 DEBUG [StoreCloser-TestAcidGuarantees,,1732486585339.8ca644d2cada6dc44b363b7383dcd4fe.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/8ca644d2cada6dc44b363b7383dcd4fe/B/5037fa9e4f23489aa5eede59338d5122 to hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/archive/data/default/TestAcidGuarantees/8ca644d2cada6dc44b363b7383dcd4fe/B/5037fa9e4f23489aa5eede59338d5122 2024-11-24T22:16:53,522 DEBUG [StoreCloser-TestAcidGuarantees,,1732486585339.8ca644d2cada6dc44b363b7383dcd4fe.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/8ca644d2cada6dc44b363b7383dcd4fe/B/f7f8edc5d3b149d5a11aa186b622f761 to hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/archive/data/default/TestAcidGuarantees/8ca644d2cada6dc44b363b7383dcd4fe/B/f7f8edc5d3b149d5a11aa186b622f761 2024-11-24T22:16:53,523 DEBUG [StoreCloser-TestAcidGuarantees,,1732486585339.8ca644d2cada6dc44b363b7383dcd4fe.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/8ca644d2cada6dc44b363b7383dcd4fe/B/31584a242fc64dfcb888a07f8cbddd9f to hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/archive/data/default/TestAcidGuarantees/8ca644d2cada6dc44b363b7383dcd4fe/B/31584a242fc64dfcb888a07f8cbddd9f 2024-11-24T22:16:53,523 DEBUG [StoreCloser-TestAcidGuarantees,,1732486585339.8ca644d2cada6dc44b363b7383dcd4fe.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/8ca644d2cada6dc44b363b7383dcd4fe/B/730efb19372148f8bdb95e42b451eb85 to hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/archive/data/default/TestAcidGuarantees/8ca644d2cada6dc44b363b7383dcd4fe/B/730efb19372148f8bdb95e42b451eb85 2024-11-24T22:16:53,524 DEBUG [StoreCloser-TestAcidGuarantees,,1732486585339.8ca644d2cada6dc44b363b7383dcd4fe.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/8ca644d2cada6dc44b363b7383dcd4fe/B/6e64a39e7a474ab9a6e384c6fbeb34d3 to hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/archive/data/default/TestAcidGuarantees/8ca644d2cada6dc44b363b7383dcd4fe/B/6e64a39e7a474ab9a6e384c6fbeb34d3 2024-11-24T22:16:53,525 DEBUG [StoreCloser-TestAcidGuarantees,,1732486585339.8ca644d2cada6dc44b363b7383dcd4fe.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/8ca644d2cada6dc44b363b7383dcd4fe/B/0a9cb8108ab84a25a32039517a52cbeb to hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/archive/data/default/TestAcidGuarantees/8ca644d2cada6dc44b363b7383dcd4fe/B/0a9cb8108ab84a25a32039517a52cbeb 2024-11-24T22:16:53,526 DEBUG [StoreCloser-TestAcidGuarantees,,1732486585339.8ca644d2cada6dc44b363b7383dcd4fe.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/8ca644d2cada6dc44b363b7383dcd4fe/B/dc23ac5e77d74dc0906774d8a726505d to hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/archive/data/default/TestAcidGuarantees/8ca644d2cada6dc44b363b7383dcd4fe/B/dc23ac5e77d74dc0906774d8a726505d 2024-11-24T22:16:53,526 DEBUG [StoreCloser-TestAcidGuarantees,,1732486585339.8ca644d2cada6dc44b363b7383dcd4fe.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/8ca644d2cada6dc44b363b7383dcd4fe/B/63e6053a21424b89a81ca99ff0a18861 to hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/archive/data/default/TestAcidGuarantees/8ca644d2cada6dc44b363b7383dcd4fe/B/63e6053a21424b89a81ca99ff0a18861 2024-11-24T22:16:53,527 DEBUG [StoreCloser-TestAcidGuarantees,,1732486585339.8ca644d2cada6dc44b363b7383dcd4fe.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/8ca644d2cada6dc44b363b7383dcd4fe/B/6b52569f4db642aebd605b9ea3871fa5 to hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/archive/data/default/TestAcidGuarantees/8ca644d2cada6dc44b363b7383dcd4fe/B/6b52569f4db642aebd605b9ea3871fa5 2024-11-24T22:16:53,528 DEBUG [StoreCloser-TestAcidGuarantees,,1732486585339.8ca644d2cada6dc44b363b7383dcd4fe.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/8ca644d2cada6dc44b363b7383dcd4fe/B/e868df9854d3473fa99b4edb9eb7e4b9 to hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/archive/data/default/TestAcidGuarantees/8ca644d2cada6dc44b363b7383dcd4fe/B/e868df9854d3473fa99b4edb9eb7e4b9 2024-11-24T22:16:53,529 DEBUG [StoreCloser-TestAcidGuarantees,,1732486585339.8ca644d2cada6dc44b363b7383dcd4fe.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/8ca644d2cada6dc44b363b7383dcd4fe/B/3b068b24fdb74c6190094f2727449194 to hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/archive/data/default/TestAcidGuarantees/8ca644d2cada6dc44b363b7383dcd4fe/B/3b068b24fdb74c6190094f2727449194 2024-11-24T22:16:53,529 DEBUG [StoreCloser-TestAcidGuarantees,,1732486585339.8ca644d2cada6dc44b363b7383dcd4fe.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/8ca644d2cada6dc44b363b7383dcd4fe/B/1eaf0718bad24a90ab90b86f3a35f33f to hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/archive/data/default/TestAcidGuarantees/8ca644d2cada6dc44b363b7383dcd4fe/B/1eaf0718bad24a90ab90b86f3a35f33f 2024-11-24T22:16:53,530 DEBUG [StoreCloser-TestAcidGuarantees,,1732486585339.8ca644d2cada6dc44b363b7383dcd4fe.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/8ca644d2cada6dc44b363b7383dcd4fe/B/60a721145bed4ac1ac6c4ee15134ccdb to hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/archive/data/default/TestAcidGuarantees/8ca644d2cada6dc44b363b7383dcd4fe/B/60a721145bed4ac1ac6c4ee15134ccdb 2024-11-24T22:16:53,531 DEBUG [StoreCloser-TestAcidGuarantees,,1732486585339.8ca644d2cada6dc44b363b7383dcd4fe.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/8ca644d2cada6dc44b363b7383dcd4fe/B/8b5d192284234dd797b73ec2a04209f8 to hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/archive/data/default/TestAcidGuarantees/8ca644d2cada6dc44b363b7383dcd4fe/B/8b5d192284234dd797b73ec2a04209f8 2024-11-24T22:16:53,532 DEBUG [StoreCloser-TestAcidGuarantees,,1732486585339.8ca644d2cada6dc44b363b7383dcd4fe.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/8ca644d2cada6dc44b363b7383dcd4fe/B/bfe3e61f56434ab781a4929b3389a3c6 to hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/archive/data/default/TestAcidGuarantees/8ca644d2cada6dc44b363b7383dcd4fe/B/bfe3e61f56434ab781a4929b3389a3c6 2024-11-24T22:16:53,533 DEBUG [StoreCloser-TestAcidGuarantees,,1732486585339.8ca644d2cada6dc44b363b7383dcd4fe.-1 {}] regionserver.HStore(2317): Moving the files [hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/8ca644d2cada6dc44b363b7383dcd4fe/C/073e5ad8128f4a9fa041c49b3dd915d6, hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/8ca644d2cada6dc44b363b7383dcd4fe/C/63f9798f78214b7ab82aae257603936a, hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/8ca644d2cada6dc44b363b7383dcd4fe/C/77cfe1f3e45a4ae4a303695fd6b77f4a, hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/8ca644d2cada6dc44b363b7383dcd4fe/C/4739455ff5084251ac728987a900105a, hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/8ca644d2cada6dc44b363b7383dcd4fe/C/f1b82da7ecb14b54b6c5cd9a43234c81, hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/8ca644d2cada6dc44b363b7383dcd4fe/C/207460af5f1c459cbc35febbfcc81050, hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/8ca644d2cada6dc44b363b7383dcd4fe/C/fbb52ae7a00f44df87fcf504039c6b4a, hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/8ca644d2cada6dc44b363b7383dcd4fe/C/e8487447b7fe43cb9ab44acd74a6b5be, hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/8ca644d2cada6dc44b363b7383dcd4fe/C/4c146cfb566b43de8d7bda68c062baa8, hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/8ca644d2cada6dc44b363b7383dcd4fe/C/83fb5cbe634d494794af17bfe57a70f5, hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/8ca644d2cada6dc44b363b7383dcd4fe/C/0fec38d8c31a41a8a5233d65261f9229, hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/8ca644d2cada6dc44b363b7383dcd4fe/C/09a368f67227449abf941025f6c40c9f, hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/8ca644d2cada6dc44b363b7383dcd4fe/C/0757bc93a25a45b7a830589a9f99e3d8, hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/8ca644d2cada6dc44b363b7383dcd4fe/C/b6da2cc25fa44492ad282381b9a45c04, hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/8ca644d2cada6dc44b363b7383dcd4fe/C/ef06905113514929b387959c8588fb32, hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/8ca644d2cada6dc44b363b7383dcd4fe/C/c86f7cc086c0485b914cc3b9e5711865, hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/8ca644d2cada6dc44b363b7383dcd4fe/C/943ba0f17f0c449e98e26c922ce03508, hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/8ca644d2cada6dc44b363b7383dcd4fe/C/05a2a98b5f2043a6a93bfa785195587f, hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/8ca644d2cada6dc44b363b7383dcd4fe/C/ae16909713a3403090f5a0ac150c170a, hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/8ca644d2cada6dc44b363b7383dcd4fe/C/a88236e6fd224409ad07b616601f72ff, hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/8ca644d2cada6dc44b363b7383dcd4fe/C/d5f5e7c1ceb141538808728969043dc6, hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/8ca644d2cada6dc44b363b7383dcd4fe/C/2795ef0eb8634c1a81eabaac53d9372a, hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/8ca644d2cada6dc44b363b7383dcd4fe/C/40a953156e07441eb8fc6867ae9b1954] to archive 2024-11-24T22:16:53,533 DEBUG [StoreCloser-TestAcidGuarantees,,1732486585339.8ca644d2cada6dc44b363b7383dcd4fe.-1 {}] backup.HFileArchiver(360): Archiving compacted files. 2024-11-24T22:16:53,534 DEBUG [StoreCloser-TestAcidGuarantees,,1732486585339.8ca644d2cada6dc44b363b7383dcd4fe.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/8ca644d2cada6dc44b363b7383dcd4fe/C/073e5ad8128f4a9fa041c49b3dd915d6 to hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/archive/data/default/TestAcidGuarantees/8ca644d2cada6dc44b363b7383dcd4fe/C/073e5ad8128f4a9fa041c49b3dd915d6 2024-11-24T22:16:53,535 DEBUG [StoreCloser-TestAcidGuarantees,,1732486585339.8ca644d2cada6dc44b363b7383dcd4fe.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/8ca644d2cada6dc44b363b7383dcd4fe/C/63f9798f78214b7ab82aae257603936a to hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/archive/data/default/TestAcidGuarantees/8ca644d2cada6dc44b363b7383dcd4fe/C/63f9798f78214b7ab82aae257603936a 2024-11-24T22:16:53,536 DEBUG [StoreCloser-TestAcidGuarantees,,1732486585339.8ca644d2cada6dc44b363b7383dcd4fe.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/8ca644d2cada6dc44b363b7383dcd4fe/C/77cfe1f3e45a4ae4a303695fd6b77f4a to hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/archive/data/default/TestAcidGuarantees/8ca644d2cada6dc44b363b7383dcd4fe/C/77cfe1f3e45a4ae4a303695fd6b77f4a 2024-11-24T22:16:53,537 DEBUG [StoreCloser-TestAcidGuarantees,,1732486585339.8ca644d2cada6dc44b363b7383dcd4fe.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/8ca644d2cada6dc44b363b7383dcd4fe/C/4739455ff5084251ac728987a900105a to hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/archive/data/default/TestAcidGuarantees/8ca644d2cada6dc44b363b7383dcd4fe/C/4739455ff5084251ac728987a900105a 2024-11-24T22:16:53,537 DEBUG [StoreCloser-TestAcidGuarantees,,1732486585339.8ca644d2cada6dc44b363b7383dcd4fe.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/8ca644d2cada6dc44b363b7383dcd4fe/C/f1b82da7ecb14b54b6c5cd9a43234c81 to hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/archive/data/default/TestAcidGuarantees/8ca644d2cada6dc44b363b7383dcd4fe/C/f1b82da7ecb14b54b6c5cd9a43234c81 2024-11-24T22:16:53,538 DEBUG [StoreCloser-TestAcidGuarantees,,1732486585339.8ca644d2cada6dc44b363b7383dcd4fe.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/8ca644d2cada6dc44b363b7383dcd4fe/C/207460af5f1c459cbc35febbfcc81050 to hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/archive/data/default/TestAcidGuarantees/8ca644d2cada6dc44b363b7383dcd4fe/C/207460af5f1c459cbc35febbfcc81050 2024-11-24T22:16:53,539 DEBUG [StoreCloser-TestAcidGuarantees,,1732486585339.8ca644d2cada6dc44b363b7383dcd4fe.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/8ca644d2cada6dc44b363b7383dcd4fe/C/fbb52ae7a00f44df87fcf504039c6b4a to hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/archive/data/default/TestAcidGuarantees/8ca644d2cada6dc44b363b7383dcd4fe/C/fbb52ae7a00f44df87fcf504039c6b4a 2024-11-24T22:16:53,540 DEBUG [StoreCloser-TestAcidGuarantees,,1732486585339.8ca644d2cada6dc44b363b7383dcd4fe.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/8ca644d2cada6dc44b363b7383dcd4fe/C/e8487447b7fe43cb9ab44acd74a6b5be to hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/archive/data/default/TestAcidGuarantees/8ca644d2cada6dc44b363b7383dcd4fe/C/e8487447b7fe43cb9ab44acd74a6b5be 2024-11-24T22:16:53,540 DEBUG [StoreCloser-TestAcidGuarantees,,1732486585339.8ca644d2cada6dc44b363b7383dcd4fe.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/8ca644d2cada6dc44b363b7383dcd4fe/C/4c146cfb566b43de8d7bda68c062baa8 to hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/archive/data/default/TestAcidGuarantees/8ca644d2cada6dc44b363b7383dcd4fe/C/4c146cfb566b43de8d7bda68c062baa8 2024-11-24T22:16:53,541 DEBUG [StoreCloser-TestAcidGuarantees,,1732486585339.8ca644d2cada6dc44b363b7383dcd4fe.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/8ca644d2cada6dc44b363b7383dcd4fe/C/83fb5cbe634d494794af17bfe57a70f5 to hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/archive/data/default/TestAcidGuarantees/8ca644d2cada6dc44b363b7383dcd4fe/C/83fb5cbe634d494794af17bfe57a70f5 2024-11-24T22:16:53,542 DEBUG [StoreCloser-TestAcidGuarantees,,1732486585339.8ca644d2cada6dc44b363b7383dcd4fe.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/8ca644d2cada6dc44b363b7383dcd4fe/C/0fec38d8c31a41a8a5233d65261f9229 to hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/archive/data/default/TestAcidGuarantees/8ca644d2cada6dc44b363b7383dcd4fe/C/0fec38d8c31a41a8a5233d65261f9229 2024-11-24T22:16:53,543 DEBUG [StoreCloser-TestAcidGuarantees,,1732486585339.8ca644d2cada6dc44b363b7383dcd4fe.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/8ca644d2cada6dc44b363b7383dcd4fe/C/09a368f67227449abf941025f6c40c9f to hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/archive/data/default/TestAcidGuarantees/8ca644d2cada6dc44b363b7383dcd4fe/C/09a368f67227449abf941025f6c40c9f 2024-11-24T22:16:53,543 DEBUG [StoreCloser-TestAcidGuarantees,,1732486585339.8ca644d2cada6dc44b363b7383dcd4fe.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/8ca644d2cada6dc44b363b7383dcd4fe/C/0757bc93a25a45b7a830589a9f99e3d8 to hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/archive/data/default/TestAcidGuarantees/8ca644d2cada6dc44b363b7383dcd4fe/C/0757bc93a25a45b7a830589a9f99e3d8 2024-11-24T22:16:53,544 DEBUG [StoreCloser-TestAcidGuarantees,,1732486585339.8ca644d2cada6dc44b363b7383dcd4fe.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/8ca644d2cada6dc44b363b7383dcd4fe/C/b6da2cc25fa44492ad282381b9a45c04 to hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/archive/data/default/TestAcidGuarantees/8ca644d2cada6dc44b363b7383dcd4fe/C/b6da2cc25fa44492ad282381b9a45c04 2024-11-24T22:16:53,545 DEBUG [StoreCloser-TestAcidGuarantees,,1732486585339.8ca644d2cada6dc44b363b7383dcd4fe.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/8ca644d2cada6dc44b363b7383dcd4fe/C/ef06905113514929b387959c8588fb32 to hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/archive/data/default/TestAcidGuarantees/8ca644d2cada6dc44b363b7383dcd4fe/C/ef06905113514929b387959c8588fb32 2024-11-24T22:16:53,546 DEBUG [StoreCloser-TestAcidGuarantees,,1732486585339.8ca644d2cada6dc44b363b7383dcd4fe.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/8ca644d2cada6dc44b363b7383dcd4fe/C/c86f7cc086c0485b914cc3b9e5711865 to hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/archive/data/default/TestAcidGuarantees/8ca644d2cada6dc44b363b7383dcd4fe/C/c86f7cc086c0485b914cc3b9e5711865 2024-11-24T22:16:53,547 DEBUG [StoreCloser-TestAcidGuarantees,,1732486585339.8ca644d2cada6dc44b363b7383dcd4fe.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/8ca644d2cada6dc44b363b7383dcd4fe/C/943ba0f17f0c449e98e26c922ce03508 to hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/archive/data/default/TestAcidGuarantees/8ca644d2cada6dc44b363b7383dcd4fe/C/943ba0f17f0c449e98e26c922ce03508 2024-11-24T22:16:53,547 DEBUG [StoreCloser-TestAcidGuarantees,,1732486585339.8ca644d2cada6dc44b363b7383dcd4fe.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/8ca644d2cada6dc44b363b7383dcd4fe/C/05a2a98b5f2043a6a93bfa785195587f to hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/archive/data/default/TestAcidGuarantees/8ca644d2cada6dc44b363b7383dcd4fe/C/05a2a98b5f2043a6a93bfa785195587f 2024-11-24T22:16:53,548 DEBUG [StoreCloser-TestAcidGuarantees,,1732486585339.8ca644d2cada6dc44b363b7383dcd4fe.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/8ca644d2cada6dc44b363b7383dcd4fe/C/ae16909713a3403090f5a0ac150c170a to hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/archive/data/default/TestAcidGuarantees/8ca644d2cada6dc44b363b7383dcd4fe/C/ae16909713a3403090f5a0ac150c170a 2024-11-24T22:16:53,549 DEBUG [StoreCloser-TestAcidGuarantees,,1732486585339.8ca644d2cada6dc44b363b7383dcd4fe.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/8ca644d2cada6dc44b363b7383dcd4fe/C/a88236e6fd224409ad07b616601f72ff to hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/archive/data/default/TestAcidGuarantees/8ca644d2cada6dc44b363b7383dcd4fe/C/a88236e6fd224409ad07b616601f72ff 2024-11-24T22:16:53,550 DEBUG [StoreCloser-TestAcidGuarantees,,1732486585339.8ca644d2cada6dc44b363b7383dcd4fe.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/8ca644d2cada6dc44b363b7383dcd4fe/C/d5f5e7c1ceb141538808728969043dc6 to hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/archive/data/default/TestAcidGuarantees/8ca644d2cada6dc44b363b7383dcd4fe/C/d5f5e7c1ceb141538808728969043dc6 2024-11-24T22:16:53,550 DEBUG [StoreCloser-TestAcidGuarantees,,1732486585339.8ca644d2cada6dc44b363b7383dcd4fe.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/8ca644d2cada6dc44b363b7383dcd4fe/C/2795ef0eb8634c1a81eabaac53d9372a to hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/archive/data/default/TestAcidGuarantees/8ca644d2cada6dc44b363b7383dcd4fe/C/2795ef0eb8634c1a81eabaac53d9372a 2024-11-24T22:16:53,551 DEBUG [StoreCloser-TestAcidGuarantees,,1732486585339.8ca644d2cada6dc44b363b7383dcd4fe.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/8ca644d2cada6dc44b363b7383dcd4fe/C/40a953156e07441eb8fc6867ae9b1954 to hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/archive/data/default/TestAcidGuarantees/8ca644d2cada6dc44b363b7383dcd4fe/C/40a953156e07441eb8fc6867ae9b1954 2024-11-24T22:16:53,554 DEBUG [RS_CLOSE_REGION-regionserver/bd6592c37bb2:0-0 {event_type=M_RS_CLOSE_REGION, pid=171}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/8ca644d2cada6dc44b363b7383dcd4fe/recovered.edits/1029.seqid, newMaxSeqId=1029, maxSeqId=4 2024-11-24T22:16:53,555 INFO [RS_CLOSE_REGION-regionserver/bd6592c37bb2:0-0 {event_type=M_RS_CLOSE_REGION, pid=171}] regionserver.HRegion(1973): Closed TestAcidGuarantees,,1732486585339.8ca644d2cada6dc44b363b7383dcd4fe. 2024-11-24T22:16:53,555 DEBUG [RS_CLOSE_REGION-regionserver/bd6592c37bb2:0-0 {event_type=M_RS_CLOSE_REGION, pid=171}] regionserver.HRegion(1676): Region close journal for 8ca644d2cada6dc44b363b7383dcd4fe: Waiting for close lock at 1732486611852Running coprocessor pre-close hooks at 1732486611852Disabling compacts and flushes for region at 1732486611852Disabling writes for close at 1732486611852Obtaining lock to block concurrent updates at 1732486611852Preparing flush snapshotting stores in 8ca644d2cada6dc44b363b7383dcd4fe at 1732486611852Finished memstore snapshotting TestAcidGuarantees,,1732486585339.8ca644d2cada6dc44b363b7383dcd4fe., syncing WAL and waiting on mvcc, flushsize=dataSize=34350, getHeapSize=102720, getOffHeapSize=0, getCellsCount=750 at 1732486611853 (+1 ms)Flushing stores of TestAcidGuarantees,,1732486585339.8ca644d2cada6dc44b363b7383dcd4fe. at 1732486611853Flushing 8ca644d2cada6dc44b363b7383dcd4fe/A: creating writer at 1732486611853Flushing mob file 8ca644d2cada6dc44b363b7383dcd4fe/A: appending metadata at 1732486611857 (+4 ms)Flushing mob file 8ca644d2cada6dc44b363b7383dcd4fe/A: closing flushed file at 1732486611857Flushing 8ca644d2cada6dc44b363b7383dcd4fe/A: appending metadata at 1732486612264 (+407 ms)Flushing 8ca644d2cada6dc44b363b7383dcd4fe/A: closing flushed file at 1732486612264Flushing 8ca644d2cada6dc44b363b7383dcd4fe/B: creating writer at 1732486612670 (+406 ms)Flushing 8ca644d2cada6dc44b363b7383dcd4fe/B: appending metadata at 1732486612672 (+2 ms)Flushing 8ca644d2cada6dc44b363b7383dcd4fe/B: closing flushed file at 1732486612673 (+1 ms)Flushing 8ca644d2cada6dc44b363b7383dcd4fe/C: creating writer at 1732486613079 (+406 ms)Flushing 8ca644d2cada6dc44b363b7383dcd4fe/C: appending metadata at 1732486613081 (+2 ms)Flushing 8ca644d2cada6dc44b363b7383dcd4fe/C: closing flushed file at 1732486613081Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@4fde1af2: reopening flushed file at 1732486613487 (+406 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@129af886: reopening flushed file at 1732486613490 (+3 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@20c80e29: reopening flushed file at 1732486613493 (+3 ms)Finished flush of dataSize ~33.54 KB/34350, heapSize ~100.31 KB/102720, currentSize=0 B/0 for 8ca644d2cada6dc44b363b7383dcd4fe in 1644ms, sequenceid=1026, compaction requested=true at 1732486613496 (+3 ms)Writing region close event to WAL at 1732486613552 (+56 ms)Running coprocessor post-close hooks at 1732486613555 (+3 ms)Closed at 1732486613555 2024-11-24T22:16:53,556 INFO [RS_CLOSE_REGION-regionserver/bd6592c37bb2:0-0 {event_type=M_RS_CLOSE_REGION, pid=171}] handler.UnassignRegionHandler(157): Closed 8ca644d2cada6dc44b363b7383dcd4fe 2024-11-24T22:16:53,557 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=170 updating hbase:meta row=8ca644d2cada6dc44b363b7383dcd4fe, regionState=CLOSED 2024-11-24T22:16:53,558 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=171, ppid=170, state=RUNNABLE, hasLock=false; CloseRegionProcedure 8ca644d2cada6dc44b363b7383dcd4fe, server=bd6592c37bb2,40087,1732486423713 because future has completed 2024-11-24T22:16:53,559 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=171, resume processing ppid=170 2024-11-24T22:16:53,559 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=171, ppid=170, state=SUCCESS, hasLock=false; CloseRegionProcedure 8ca644d2cada6dc44b363b7383dcd4fe, server=bd6592c37bb2,40087,1732486423713 in 1.8580 sec 2024-11-24T22:16:53,561 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=170, resume processing ppid=169 2024-11-24T22:16:53,561 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=170, ppid=169, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=8ca644d2cada6dc44b363b7383dcd4fe, UNASSIGN in 1.8610 sec 2024-11-24T22:16:53,562 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=169, resume processing ppid=168 2024-11-24T22:16:53,562 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=169, ppid=168, state=SUCCESS, hasLock=false; CloseTableRegionsProcedure table=TestAcidGuarantees in 1.8640 sec 2024-11-24T22:16:53,563 DEBUG [PEWorker-5 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1732486613563"}]},"ts":"1732486613563"} 2024-11-24T22:16:53,564 INFO [PEWorker-5 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=DISABLED in hbase:meta 2024-11-24T22:16:53,564 INFO [PEWorker-5 {}] procedure.DisableTableProcedure(296): Set TestAcidGuarantees to state=DISABLED 2024-11-24T22:16:53,566 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=168, state=SUCCESS, hasLock=false; DisableTableProcedure table=TestAcidGuarantees in 1.8720 sec 2024-11-24T22:16:53,851 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37701 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=168 2024-11-24T22:16:53,851 INFO [RPCClient-NioEventLoopGroup-4-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: DISABLE, Table Name: default:TestAcidGuarantees completed 2024-11-24T22:16:53,851 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37701 {}] master.HMaster$5(2570): Client=jenkins//172.17.0.2 delete TestAcidGuarantees 2024-11-24T22:16:53,852 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37701 {}] procedure2.ProcedureExecutor(1139): Stored pid=172, state=RUNNABLE:DELETE_TABLE_PRE_OPERATION, hasLock=false; DeleteTableProcedure table=TestAcidGuarantees 2024-11-24T22:16:53,853 DEBUG [PEWorker-1 {}] procedure.DeleteTableProcedure(103): Waiting for RIT for pid=172, state=RUNNABLE:DELETE_TABLE_PRE_OPERATION, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-24T22:16:53,853 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37701 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=172 2024-11-24T22:16:53,853 DEBUG [PEWorker-1 {}] procedure.DeleteTableProcedure(115): Deleting regions from filesystem for pid=172, state=RUNNABLE:DELETE_TABLE_CLEAR_FS_LAYOUT, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-24T22:16:53,855 DEBUG [HFileArchiver-6 {}] backup.HFileArchiver(131): ARCHIVING hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/8ca644d2cada6dc44b363b7383dcd4fe 2024-11-24T22:16:53,857 DEBUG [HFileArchiver-6 {}] backup.HFileArchiver(159): Archiving [FileablePath, hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/8ca644d2cada6dc44b363b7383dcd4fe/A, FileablePath, hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/8ca644d2cada6dc44b363b7383dcd4fe/B, FileablePath, hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/8ca644d2cada6dc44b363b7383dcd4fe/C, FileablePath, hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/8ca644d2cada6dc44b363b7383dcd4fe/recovered.edits] 2024-11-24T22:16:53,859 DEBUG [HFileArchiver-6 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/8ca644d2cada6dc44b363b7383dcd4fe/A/0d373842ebe64020a36578cdb9bbe1bf to hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/archive/data/default/TestAcidGuarantees/8ca644d2cada6dc44b363b7383dcd4fe/A/0d373842ebe64020a36578cdb9bbe1bf 2024-11-24T22:16:53,859 DEBUG [HFileArchiver-6 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/8ca644d2cada6dc44b363b7383dcd4fe/A/3589d762efd14116bfc39bff83dbd9d9 to hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/archive/data/default/TestAcidGuarantees/8ca644d2cada6dc44b363b7383dcd4fe/A/3589d762efd14116bfc39bff83dbd9d9 2024-11-24T22:16:53,860 DEBUG [HFileArchiver-6 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/8ca644d2cada6dc44b363b7383dcd4fe/A/6b8970e0d95a497095d0ced45e8bb8b2 to hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/archive/data/default/TestAcidGuarantees/8ca644d2cada6dc44b363b7383dcd4fe/A/6b8970e0d95a497095d0ced45e8bb8b2 2024-11-24T22:16:53,861 DEBUG [HFileArchiver-6 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/8ca644d2cada6dc44b363b7383dcd4fe/A/f30a3ace6b014020ac8f8da210794c6c to hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/archive/data/default/TestAcidGuarantees/8ca644d2cada6dc44b363b7383dcd4fe/A/f30a3ace6b014020ac8f8da210794c6c 2024-11-24T22:16:53,862 DEBUG [HFileArchiver-6 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/8ca644d2cada6dc44b363b7383dcd4fe/B/9986550607554199af403f20b3ac0c89 to hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/archive/data/default/TestAcidGuarantees/8ca644d2cada6dc44b363b7383dcd4fe/B/9986550607554199af403f20b3ac0c89 2024-11-24T22:16:53,863 DEBUG [HFileArchiver-6 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/8ca644d2cada6dc44b363b7383dcd4fe/B/c6770c17f5d84d469ece27b38636bc53 to hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/archive/data/default/TestAcidGuarantees/8ca644d2cada6dc44b363b7383dcd4fe/B/c6770c17f5d84d469ece27b38636bc53 2024-11-24T22:16:53,864 DEBUG [HFileArchiver-6 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/8ca644d2cada6dc44b363b7383dcd4fe/B/ceca785ebc18479b946c77f66d69cb33 to hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/archive/data/default/TestAcidGuarantees/8ca644d2cada6dc44b363b7383dcd4fe/B/ceca785ebc18479b946c77f66d69cb33 2024-11-24T22:16:53,864 DEBUG [HFileArchiver-6 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/8ca644d2cada6dc44b363b7383dcd4fe/B/ed0b458c5c334334a162cfbf3d0063e1 to hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/archive/data/default/TestAcidGuarantees/8ca644d2cada6dc44b363b7383dcd4fe/B/ed0b458c5c334334a162cfbf3d0063e1 2024-11-24T22:16:53,866 DEBUG [HFileArchiver-6 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/8ca644d2cada6dc44b363b7383dcd4fe/C/0f7897c4877f4efbbdefd8c788913552 to hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/archive/data/default/TestAcidGuarantees/8ca644d2cada6dc44b363b7383dcd4fe/C/0f7897c4877f4efbbdefd8c788913552 2024-11-24T22:16:53,866 DEBUG [HFileArchiver-6 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/8ca644d2cada6dc44b363b7383dcd4fe/C/90980d39589642b19b8884b4423eed49 to hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/archive/data/default/TestAcidGuarantees/8ca644d2cada6dc44b363b7383dcd4fe/C/90980d39589642b19b8884b4423eed49 2024-11-24T22:16:53,867 DEBUG [HFileArchiver-6 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/8ca644d2cada6dc44b363b7383dcd4fe/C/9b5141abb9cf461b843e321b6ea342e9 to hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/archive/data/default/TestAcidGuarantees/8ca644d2cada6dc44b363b7383dcd4fe/C/9b5141abb9cf461b843e321b6ea342e9 2024-11-24T22:16:53,868 DEBUG [HFileArchiver-6 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/8ca644d2cada6dc44b363b7383dcd4fe/C/a020e316909c4bb7955baa0097597ff8 to hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/archive/data/default/TestAcidGuarantees/8ca644d2cada6dc44b363b7383dcd4fe/C/a020e316909c4bb7955baa0097597ff8 2024-11-24T22:16:53,869 DEBUG [HFileArchiver-6 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/8ca644d2cada6dc44b363b7383dcd4fe/recovered.edits/1029.seqid to hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/archive/data/default/TestAcidGuarantees/8ca644d2cada6dc44b363b7383dcd4fe/recovered.edits/1029.seqid 2024-11-24T22:16:53,870 DEBUG [HFileArchiver-6 {}] backup.HFileArchiver(610): Deleted hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/default/TestAcidGuarantees/8ca644d2cada6dc44b363b7383dcd4fe 2024-11-24T22:16:53,870 DEBUG [PEWorker-1 {}] procedure.DeleteTableProcedure(313): Archived TestAcidGuarantees regions 2024-11-24T22:16:53,870 DEBUG [PEWorker-1 {}] backup.HFileArchiver(131): ARCHIVING hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3 2024-11-24T22:16:53,870 DEBUG [PEWorker-1 {}] backup.HFileArchiver(159): Archiving [FileablePath, hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A] 2024-11-24T22:16:53,872 DEBUG [PEWorker-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024112400a69d311b5b443f932c804300f5c47d_8ca644d2cada6dc44b363b7383dcd4fe to hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024112400a69d311b5b443f932c804300f5c47d_8ca644d2cada6dc44b363b7383dcd4fe 2024-11-24T22:16:53,873 DEBUG [PEWorker-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241124055bd7d48f9849e8aebf06de5bc74b92_8ca644d2cada6dc44b363b7383dcd4fe to hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241124055bd7d48f9849e8aebf06de5bc74b92_8ca644d2cada6dc44b363b7383dcd4fe 2024-11-24T22:16:53,874 DEBUG [PEWorker-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241124228b01d3b9244ae0892af72a99fd9e63_8ca644d2cada6dc44b363b7383dcd4fe to hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241124228b01d3b9244ae0892af72a99fd9e63_8ca644d2cada6dc44b363b7383dcd4fe 2024-11-24T22:16:53,874 DEBUG [PEWorker-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411243ea02dbc227041e8a709a83b3533db7a_8ca644d2cada6dc44b363b7383dcd4fe to hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411243ea02dbc227041e8a709a83b3533db7a_8ca644d2cada6dc44b363b7383dcd4fe 2024-11-24T22:16:53,875 DEBUG [PEWorker-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241124464ea015c92745a988743391450e3d30_8ca644d2cada6dc44b363b7383dcd4fe to hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241124464ea015c92745a988743391450e3d30_8ca644d2cada6dc44b363b7383dcd4fe 2024-11-24T22:16:53,876 DEBUG [PEWorker-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411244c657d2b8c0349f29eecc4bb03459c63_8ca644d2cada6dc44b363b7383dcd4fe to hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411244c657d2b8c0349f29eecc4bb03459c63_8ca644d2cada6dc44b363b7383dcd4fe 2024-11-24T22:16:53,877 DEBUG [PEWorker-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241124587d91e3431c4d7e948bb918733ef8d3_8ca644d2cada6dc44b363b7383dcd4fe to hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241124587d91e3431c4d7e948bb918733ef8d3_8ca644d2cada6dc44b363b7383dcd4fe 2024-11-24T22:16:53,877 DEBUG [PEWorker-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411248adf33bdcf2d4bad9a3cca7110343f1c_8ca644d2cada6dc44b363b7383dcd4fe to hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411248adf33bdcf2d4bad9a3cca7110343f1c_8ca644d2cada6dc44b363b7383dcd4fe 2024-11-24T22:16:53,878 DEBUG [PEWorker-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411248ec87d903bc94f8089f0ac1a67cd13d1_8ca644d2cada6dc44b363b7383dcd4fe to hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411248ec87d903bc94f8089f0ac1a67cd13d1_8ca644d2cada6dc44b363b7383dcd4fe 2024-11-24T22:16:53,879 DEBUG [PEWorker-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024112491b71ebac22b471cac5be7cc16cbdf76_8ca644d2cada6dc44b363b7383dcd4fe to hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024112491b71ebac22b471cac5be7cc16cbdf76_8ca644d2cada6dc44b363b7383dcd4fe 2024-11-24T22:16:53,880 DEBUG [PEWorker-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411249d703a0b5ce54c4398620884718cbe38_8ca644d2cada6dc44b363b7383dcd4fe to hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411249d703a0b5ce54c4398620884718cbe38_8ca644d2cada6dc44b363b7383dcd4fe 2024-11-24T22:16:53,880 DEBUG [PEWorker-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411249e1c5c735c0242908fd028413e7a39dc_8ca644d2cada6dc44b363b7383dcd4fe to hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411249e1c5c735c0242908fd028413e7a39dc_8ca644d2cada6dc44b363b7383dcd4fe 2024-11-24T22:16:53,881 DEBUG [PEWorker-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241124bda66f2b082942a998944897f1893b7f_8ca644d2cada6dc44b363b7383dcd4fe to hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241124bda66f2b082942a998944897f1893b7f_8ca644d2cada6dc44b363b7383dcd4fe 2024-11-24T22:16:53,882 DEBUG [PEWorker-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241124c28a62c49349432aaa77d1513ea0f9c4_8ca644d2cada6dc44b363b7383dcd4fe to hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241124c28a62c49349432aaa77d1513ea0f9c4_8ca644d2cada6dc44b363b7383dcd4fe 2024-11-24T22:16:53,883 DEBUG [PEWorker-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241124c462436d746b4a35bcad74286527b5c6_8ca644d2cada6dc44b363b7383dcd4fe to hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241124c462436d746b4a35bcad74286527b5c6_8ca644d2cada6dc44b363b7383dcd4fe 2024-11-24T22:16:53,883 DEBUG [PEWorker-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241124d793224a549d4d61abfabfb8ba057ae1_8ca644d2cada6dc44b363b7383dcd4fe to hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241124d793224a549d4d61abfabfb8ba057ae1_8ca644d2cada6dc44b363b7383dcd4fe 2024-11-24T22:16:53,884 DEBUG [PEWorker-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241124df325bb5e0354ee5be9c82bf6a0b1763_8ca644d2cada6dc44b363b7383dcd4fe to hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241124df325bb5e0354ee5be9c82bf6a0b1763_8ca644d2cada6dc44b363b7383dcd4fe 2024-11-24T22:16:53,885 DEBUG [PEWorker-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241124e0bdee4dd60c465bbb70df873731e23d_8ca644d2cada6dc44b363b7383dcd4fe to hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241124e0bdee4dd60c465bbb70df873731e23d_8ca644d2cada6dc44b363b7383dcd4fe 2024-11-24T22:16:53,886 DEBUG [PEWorker-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241124e959a70aeeb4453c80fd9a09da9e639b_8ca644d2cada6dc44b363b7383dcd4fe to hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241124e959a70aeeb4453c80fd9a09da9e639b_8ca644d2cada6dc44b363b7383dcd4fe 2024-11-24T22:16:53,887 DEBUG [PEWorker-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241124f9c897fa992444198f55db8a31246739_8ca644d2cada6dc44b363b7383dcd4fe to hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241124f9c897fa992444198f55db8a31246739_8ca644d2cada6dc44b363b7383dcd4fe 2024-11-24T22:16:53,887 DEBUG [PEWorker-1 {}] backup.HFileArchiver(610): Deleted hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3 2024-11-24T22:16:53,889 DEBUG [PEWorker-1 {}] procedure.DeleteTableProcedure(120): Deleting regions from META for pid=172, state=RUNNABLE:DELETE_TABLE_REMOVE_FROM_META, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-24T22:16:53,891 WARN [PEWorker-1 {}] procedure.DeleteTableProcedure(370): Deleting some vestigial 1 rows of TestAcidGuarantees from hbase:meta 2024-11-24T22:16:53,893 DEBUG [PEWorker-1 {}] procedure.DeleteTableProcedure(407): Removing 'TestAcidGuarantees' descriptor. 2024-11-24T22:16:53,894 DEBUG [PEWorker-1 {}] procedure.DeleteTableProcedure(126): Deleting assignment state for pid=172, state=RUNNABLE:DELETE_TABLE_UNASSIGN_REGIONS, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-24T22:16:53,894 DEBUG [PEWorker-1 {}] procedure.DeleteTableProcedure(397): Removing 'TestAcidGuarantees' from region states. 2024-11-24T22:16:53,894 DEBUG [PEWorker-1 {}] assignment.RegionStateStore(723): Delete {"totalColumns":1,"row":"TestAcidGuarantees,,1732486585339.8ca644d2cada6dc44b363b7383dcd4fe.","families":{"info":[{"qualifier":"","vlen":0,"tag":[],"timestamp":"1732486613894"}]},"ts":"9223372036854775807"} 2024-11-24T22:16:53,895 INFO [PEWorker-1 {}] assignment.RegionStateStore(562): Deleted 1 regions from META 2024-11-24T22:16:53,895 DEBUG [PEWorker-1 {}] assignment.RegionStateStore(563): Deleted regions: [{ENCODED => 8ca644d2cada6dc44b363b7383dcd4fe, NAME => 'TestAcidGuarantees,,1732486585339.8ca644d2cada6dc44b363b7383dcd4fe.', STARTKEY => '', ENDKEY => ''}] 2024-11-24T22:16:53,896 DEBUG [PEWorker-1 {}] procedure.DeleteTableProcedure(401): Marking 'TestAcidGuarantees' as deleted. 2024-11-24T22:16:53,896 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Delete {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":0,"tag":[],"timestamp":"1732486613896"}]},"ts":"9223372036854775807"} 2024-11-24T22:16:53,897 INFO [PEWorker-1 {}] hbase.MetaTableAccessor(867): Deleted table TestAcidGuarantees state from META 2024-11-24T22:16:53,897 DEBUG [PEWorker-1 {}] procedure.DeleteTableProcedure(133): Finished pid=172, state=RUNNABLE:DELETE_TABLE_POST_OPERATION, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-24T22:16:53,898 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=172, state=SUCCESS, hasLock=false; DeleteTableProcedure table=TestAcidGuarantees in 46 msec 2024-11-24T22:16:53,960 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37701 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=172 2024-11-24T22:16:53,961 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocator(219): Clear meta cache for TestAcidGuarantees 2024-11-24T22:16:53,961 INFO [RPCClient-NioEventLoopGroup-4-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: DELETE, Table Name: default:TestAcidGuarantees completed 2024-11-24T22:16:53,970 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: TestAcidGuaranteesWithEagerPolicy#testMobGetAtomicity Thread=251 (was 248) - Thread LEAK? -, OpenFileDescriptor=453 (was 449) - OpenFileDescriptor LEAK? -, MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=318 (was 330), ProcessCount=11 (was 11), AvailableMemoryMB=5669 (was 5785) 2024-11-24T22:16:53,970 INFO [Time-limited test {}] hbase.HBaseTestingUtil(1019): Shutting down minicluster 2024-11-24T22:16:53,970 INFO [Time-limited test {}] client.AsyncConnectionImpl(233): Connection has been closed by Time-limited test. 2024-11-24T22:16:53,970 DEBUG [Time-limited test {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hbase.thirdparty.com.google.common.io.Closeables.close(Closeables.java:79) at org.apache.hadoop.hbase.HBaseTestingUtil.closeConnection(HBaseTestingUtil.java:2611) at org.apache.hadoop.hbase.HBaseTestingUtil.cleanup(HBaseTestingUtil.java:1065) at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniHBaseCluster(HBaseTestingUtil.java:1034) at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniCluster(HBaseTestingUtil.java:1020) at org.apache.hadoop.hbase.AcidGuaranteesTestBase.tearDownAfterClass(AcidGuaranteesTestBase.java:66) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) at java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) at java.base/java.lang.reflect.Method.invoke(Method.java:568) at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) at org.junit.internal.runners.statements.RunAfters.invokeMethod(RunAfters.java:46) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:33) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) at java.base/java.util.concurrent.FutureTask.run(FutureTask.java:264) at java.base/java.lang.Thread.run(Thread.java:840) 2024-11-24T22:16:53,970 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-24T22:16:53,970 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-24T22:16:53,970 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-24T22:16:53,970 DEBUG [Time-limited test {}] util.JVMClusterUtil(237): Shutting down HBase Cluster 2024-11-24T22:16:53,970 DEBUG [Time-limited test {}] util.JVMClusterUtil(257): Found active master hash=129225742, stopped=false 2024-11-24T22:16:53,971 INFO [Time-limited test {}] master.ServerManager(983): Cluster shutdown requested of master=bd6592c37bb2,37701,1732486422992 2024-11-24T22:16:53,972 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:40087-0x1014c0ac8d60001, quorum=127.0.0.1:59994, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/running 2024-11-24T22:16:53,972 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:40087-0x1014c0ac8d60001, quorum=127.0.0.1:59994, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-24T22:16:53,972 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:37701-0x1014c0ac8d60000, quorum=127.0.0.1:59994, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/running 2024-11-24T22:16:53,972 INFO [Time-limited test {}] procedure2.ProcedureExecutor(723): Stopping 2024-11-24T22:16:53,972 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:37701-0x1014c0ac8d60000, quorum=127.0.0.1:59994, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-24T22:16:53,973 INFO [Time-limited test {}] client.AsyncConnectionImpl(233): Connection has been closed by Time-limited test. 2024-11-24T22:16:53,973 DEBUG [Time-limited test {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.master.HMaster.lambda$shutdown$17(HMaster.java:3306) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.master.HMaster.shutdown(HMaster.java:3277) at org.apache.hadoop.hbase.util.JVMClusterUtil.shutdown(JVMClusterUtil.java:265) at org.apache.hadoop.hbase.LocalHBaseCluster.shutdown(LocalHBaseCluster.java:416) at org.apache.hadoop.hbase.SingleProcessHBaseCluster.shutdown(SingleProcessHBaseCluster.java:676) at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniHBaseCluster(HBaseTestingUtil.java:1036) at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniCluster(HBaseTestingUtil.java:1020) at org.apache.hadoop.hbase.AcidGuaranteesTestBase.tearDownAfterClass(AcidGuaranteesTestBase.java:66) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) at java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) at java.base/java.lang.reflect.Method.invoke(Method.java:568) at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) at org.junit.internal.runners.statements.RunAfters.invokeMethod(RunAfters.java:46) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:33) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) at java.base/java.util.concurrent.FutureTask.run(FutureTask.java:264) at java.base/java.lang.Thread.run(Thread.java:840) 2024-11-24T22:16:53,973 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKUtil(113): regionserver:40087-0x1014c0ac8d60001, quorum=127.0.0.1:59994, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running 2024-11-24T22:16:53,973 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKUtil(113): master:37701-0x1014c0ac8d60000, quorum=127.0.0.1:59994, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running 2024-11-24T22:16:53,973 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-24T22:16:53,974 INFO [Time-limited test {}] regionserver.HRegionServer(2196): ***** STOPPING region server 'bd6592c37bb2,40087,1732486423713' ***** 2024-11-24T22:16:53,974 INFO [Time-limited test {}] regionserver.HRegionServer(2210): STOPPED: Shutdown requested 2024-11-24T22:16:53,974 INFO [RS:0;bd6592c37bb2:40087 {}] regionserver.HeapMemoryManager(220): Stopping 2024-11-24T22:16:53,974 INFO [MemStoreFlusher.0 {}] regionserver.MemStoreFlusher$FlushHandler(374): MemStoreFlusher.0 exiting 2024-11-24T22:16:53,974 INFO [RS:0;bd6592c37bb2:40087 {}] flush.RegionServerFlushTableProcedureManager(119): Stopping region server flush procedure manager gracefully. 2024-11-24T22:16:53,974 INFO [RS:0;bd6592c37bb2:40087 {}] snapshot.RegionServerSnapshotManager(137): Stopping RegionServerSnapshotManager gracefully. 2024-11-24T22:16:53,975 INFO [RS:0;bd6592c37bb2:40087 {}] regionserver.HRegionServer(959): stopping server bd6592c37bb2,40087,1732486423713 2024-11-24T22:16:53,975 INFO [RS:0;bd6592c37bb2:40087 {}] hbase.HBaseServerBase(455): Close async cluster connection 2024-11-24T22:16:53,975 INFO [RS:0;bd6592c37bb2:40087 {}] client.AsyncConnectionImpl(233): Connection has been closed by RS:0;bd6592c37bb2:40087. 2024-11-24T22:16:53,975 DEBUG [RS:0;bd6592c37bb2:40087 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.HBaseServerBase.closeClusterConnection(HBaseServerBase.java:457) at org.apache.hadoop.hbase.regionserver.HRegionServer.run(HRegionServer.java:962) at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer.runRegionServer(SingleProcessHBaseCluster.java:171) at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer$1.run(SingleProcessHBaseCluster.java:155) at java.base/java.security.AccessController.doPrivileged(AccessController.java:399) at java.base/javax.security.auth.Subject.doAs(Subject.java:376) at org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1930) at org.apache.hadoop.hbase.security.User$SecureHadoopUser.runAs(User.java:322) at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer.run(SingleProcessHBaseCluster.java:152) at java.base/java.lang.Thread.run(Thread.java:840) 2024-11-24T22:16:53,975 DEBUG [RS:0;bd6592c37bb2:40087 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-24T22:16:53,975 INFO [RS:0;bd6592c37bb2:40087 {}] regionserver.CompactSplit(469): Waiting for Split Thread to finish... 2024-11-24T22:16:53,975 INFO [RS:0;bd6592c37bb2:40087 {}] regionserver.CompactSplit(469): Waiting for Large Compaction Thread to finish... 2024-11-24T22:16:53,975 INFO [RS:0;bd6592c37bb2:40087 {}] regionserver.CompactSplit(469): Waiting for Small Compaction Thread to finish... 2024-11-24T22:16:53,975 INFO [RS:0;bd6592c37bb2:40087 {}] regionserver.HRegionServer(3091): Received CLOSE for 1588230740 2024-11-24T22:16:53,976 INFO [RS:0;bd6592c37bb2:40087 {}] regionserver.HRegionServer(1321): Waiting on 1 regions to close 2024-11-24T22:16:53,976 DEBUG [RS:0;bd6592c37bb2:40087 {}] regionserver.HRegionServer(1325): Online Regions={1588230740=hbase:meta,,1.1588230740} 2024-11-24T22:16:53,976 DEBUG [RS:0;bd6592c37bb2:40087 {}] regionserver.HRegionServer(1351): Waiting on 1588230740 2024-11-24T22:16:53,976 DEBUG [RS_CLOSE_META-regionserver/bd6592c37bb2:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1722): Closing 1588230740, disabling compactions & flushes 2024-11-24T22:16:53,976 INFO [RS_CLOSE_META-regionserver/bd6592c37bb2:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1755): Closing region hbase:meta,,1.1588230740 2024-11-24T22:16:53,976 DEBUG [RS_CLOSE_META-regionserver/bd6592c37bb2:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1776): Time limited wait for close lock on hbase:meta,,1.1588230740 2024-11-24T22:16:53,976 DEBUG [RS_CLOSE_META-regionserver/bd6592c37bb2:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1843): Acquired close lock on hbase:meta,,1.1588230740 after waiting 0 ms 2024-11-24T22:16:53,976 DEBUG [RS_CLOSE_META-regionserver/bd6592c37bb2:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1853): Updates disabled for region hbase:meta,,1.1588230740 2024-11-24T22:16:53,976 INFO [RS_CLOSE_META-regionserver/bd6592c37bb2:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(2902): Flushing 1588230740 4/4 column families, dataSize=19.91 KB heapSize=35.20 KB 2024-11-24T22:16:54,001 DEBUG [RS_CLOSE_META-regionserver/bd6592c37bb2:0-0 {event_type=M_RS_CLOSE_META}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/hbase/meta/1588230740/.tmp/info/0a1054e742194f76a93d039ebaa24a04 is 95, key is TestAcidGuarantees,,1732486426439.f0252adfc27b54a76a05585f3f45bf90./info:/1732486460573/DeleteFamily/seqid=0 2024-11-24T22:16:54,003 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45929 is added to blk_1073742494_1670 (size=6514) 2024-11-24T22:16:54,021 INFO [regionserver/bd6592c37bb2:0.leaseChecker {}] regionserver.LeaseManager(133): Closed leases 2024-11-24T22:16:54,176 DEBUG [RS:0;bd6592c37bb2:40087 {}] regionserver.HRegionServer(1351): Waiting on 1588230740 2024-11-24T22:16:54,348 INFO [regionserver/bd6592c37bb2:0.Chore.1 {}] hbase.ScheduledChore(149): Chore: MemstoreFlusherChore was stopped 2024-11-24T22:16:54,348 INFO [regionserver/bd6592c37bb2:0.Chore.1 {}] hbase.ScheduledChore(149): Chore: CompactionChecker was stopped 2024-11-24T22:16:54,376 DEBUG [RS:0;bd6592c37bb2:40087 {}] regionserver.HRegionServer(1351): Waiting on 1588230740 2024-11-24T22:16:54,404 INFO [RS_CLOSE_META-regionserver/bd6592c37bb2:0-0 {event_type=M_RS_CLOSE_META}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=16.75 KB at sequenceid=90 (bloomFilter=true), to=hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/hbase/meta/1588230740/.tmp/info/0a1054e742194f76a93d039ebaa24a04 2024-11-24T22:16:54,422 DEBUG [RS_CLOSE_META-regionserver/bd6592c37bb2:0-0 {event_type=M_RS_CLOSE_META}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/hbase/meta/1588230740/.tmp/ns/59453c293d724c60bbaeb16ef5e77e2b is 93, key is TestAcidGuarantees,,1732486426439.f0252adfc27b54a76a05585f3f45bf90./ns:/1732486460533/DeleteFamily/seqid=0 2024-11-24T22:16:54,424 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45929 is added to blk_1073742495_1671 (size=5906) 2024-11-24T22:16:54,576 DEBUG [RS:0;bd6592c37bb2:40087 {}] regionserver.HRegionServer(1351): Waiting on 1588230740 2024-11-24T22:16:54,777 DEBUG [RS:0;bd6592c37bb2:40087 {}] regionserver.HRegionServer(1351): Waiting on 1588230740 2024-11-24T22:16:54,825 INFO [RS_CLOSE_META-regionserver/bd6592c37bb2:0-0 {event_type=M_RS_CLOSE_META}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=608 B at sequenceid=90 (bloomFilter=true), to=hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/hbase/meta/1588230740/.tmp/ns/59453c293d724c60bbaeb16ef5e77e2b 2024-11-24T22:16:54,843 DEBUG [RS_CLOSE_META-regionserver/bd6592c37bb2:0-0 {event_type=M_RS_CLOSE_META}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/hbase/meta/1588230740/.tmp/rep_barrier/323e548962394eb19be3fb6fb2754800 is 102, key is TestAcidGuarantees,,1732486426439.f0252adfc27b54a76a05585f3f45bf90./rep_barrier:/1732486460533/DeleteFamily/seqid=0 2024-11-24T22:16:54,846 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45929 is added to blk_1073742496_1672 (size=6025) 2024-11-24T22:16:54,977 DEBUG [RS:0;bd6592c37bb2:40087 {}] regionserver.HRegionServer(1351): Waiting on 1588230740 2024-11-24T22:16:55,177 DEBUG [RS:0;bd6592c37bb2:40087 {}] regionserver.HRegionServer(1351): Waiting on 1588230740 2024-11-24T22:16:55,247 INFO [RS_CLOSE_META-regionserver/bd6592c37bb2:0-0 {event_type=M_RS_CLOSE_META}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=588 B at sequenceid=90 (bloomFilter=true), to=hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/hbase/meta/1588230740/.tmp/rep_barrier/323e548962394eb19be3fb6fb2754800 2024-11-24T22:16:55,264 DEBUG [RS_CLOSE_META-regionserver/bd6592c37bb2:0-0 {event_type=M_RS_CLOSE_META}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/hbase/meta/1588230740/.tmp/table/e8bd3fdd0f89432596812fea7f043f38 is 96, key is TestAcidGuarantees,,1732486426439.f0252adfc27b54a76a05585f3f45bf90./table:/1732486460533/DeleteFamily/seqid=0 2024-11-24T22:16:55,267 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45929 is added to blk_1073742497_1673 (size=5936) 2024-11-24T22:16:55,377 DEBUG [RS:0;bd6592c37bb2:40087 {}] regionserver.HRegionServer(1351): Waiting on 1588230740 2024-11-24T22:16:55,577 DEBUG [RS:0;bd6592c37bb2:40087 {}] regionserver.HRegionServer(1351): Waiting on 1588230740 2024-11-24T22:16:55,668 INFO [RS_CLOSE_META-regionserver/bd6592c37bb2:0-0 {event_type=M_RS_CLOSE_META}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=1.99 KB at sequenceid=90 (bloomFilter=true), to=hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/hbase/meta/1588230740/.tmp/table/e8bd3fdd0f89432596812fea7f043f38 2024-11-24T22:16:55,671 DEBUG [RS_CLOSE_META-regionserver/bd6592c37bb2:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/hbase/meta/1588230740/.tmp/info/0a1054e742194f76a93d039ebaa24a04 as hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/hbase/meta/1588230740/info/0a1054e742194f76a93d039ebaa24a04 2024-11-24T22:16:55,673 INFO [RS_CLOSE_META-regionserver/bd6592c37bb2:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/hbase/meta/1588230740/info/0a1054e742194f76a93d039ebaa24a04, entries=12, sequenceid=90, filesize=6.4 K 2024-11-24T22:16:55,674 DEBUG [RS_CLOSE_META-regionserver/bd6592c37bb2:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/hbase/meta/1588230740/.tmp/ns/59453c293d724c60bbaeb16ef5e77e2b as hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/hbase/meta/1588230740/ns/59453c293d724c60bbaeb16ef5e77e2b 2024-11-24T22:16:55,676 INFO [RS_CLOSE_META-regionserver/bd6592c37bb2:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/hbase/meta/1588230740/ns/59453c293d724c60bbaeb16ef5e77e2b, entries=8, sequenceid=90, filesize=5.8 K 2024-11-24T22:16:55,677 DEBUG [RS_CLOSE_META-regionserver/bd6592c37bb2:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/hbase/meta/1588230740/.tmp/rep_barrier/323e548962394eb19be3fb6fb2754800 as hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/hbase/meta/1588230740/rep_barrier/323e548962394eb19be3fb6fb2754800 2024-11-24T22:16:55,679 INFO [RS_CLOSE_META-regionserver/bd6592c37bb2:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/hbase/meta/1588230740/rep_barrier/323e548962394eb19be3fb6fb2754800, entries=6, sequenceid=90, filesize=5.9 K 2024-11-24T22:16:55,680 DEBUG [RS_CLOSE_META-regionserver/bd6592c37bb2:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/hbase/meta/1588230740/.tmp/table/e8bd3fdd0f89432596812fea7f043f38 as hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/hbase/meta/1588230740/table/e8bd3fdd0f89432596812fea7f043f38 2024-11-24T22:16:55,682 INFO [RS_CLOSE_META-regionserver/bd6592c37bb2:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/hbase/meta/1588230740/table/e8bd3fdd0f89432596812fea7f043f38, entries=7, sequenceid=90, filesize=5.8 K 2024-11-24T22:16:55,682 INFO [RS_CLOSE_META-regionserver/bd6592c37bb2:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(3140): Finished flush of dataSize ~19.91 KB/20384, heapSize ~35.14 KB/35984, currentSize=0 B/0 for 1588230740 in 1706ms, sequenceid=90, compaction requested=false 2024-11-24T22:16:55,686 DEBUG [RS_CLOSE_META-regionserver/bd6592c37bb2:0-0 {event_type=M_RS_CLOSE_META}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/data/hbase/meta/1588230740/recovered.edits/93.seqid, newMaxSeqId=93, maxSeqId=1 2024-11-24T22:16:55,686 DEBUG [RS_CLOSE_META-regionserver/bd6592c37bb2:0-0 {event_type=M_RS_CLOSE_META}] coprocessor.CoprocessorHost(310): Stop coprocessor org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint 2024-11-24T22:16:55,687 INFO [RS_CLOSE_META-regionserver/bd6592c37bb2:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1973): Closed hbase:meta,,1.1588230740 2024-11-24T22:16:55,687 DEBUG [RS_CLOSE_META-regionserver/bd6592c37bb2:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1676): Region close journal for 1588230740: Waiting for close lock at 1732486613976Running coprocessor pre-close hooks at 1732486613976Disabling compacts and flushes for region at 1732486613976Disabling writes for close at 1732486613976Obtaining lock to block concurrent updates at 1732486613976Preparing flush snapshotting stores in 1588230740 at 1732486613976Finished memstore snapshotting hbase:meta,,1.1588230740, syncing WAL and waiting on mvcc, flushsize=dataSize=20384, getHeapSize=35984, getOffHeapSize=0, getCellsCount=191 at 1732486613977 (+1 ms)Flushing stores of hbase:meta,,1.1588230740 at 1732486613977Flushing 1588230740/info: creating writer at 1732486613978 (+1 ms)Flushing 1588230740/info: appending metadata at 1732486614000 (+22 ms)Flushing 1588230740/info: closing flushed file at 1732486614000Flushing 1588230740/ns: creating writer at 1732486614407 (+407 ms)Flushing 1588230740/ns: appending metadata at 1732486614421 (+14 ms)Flushing 1588230740/ns: closing flushed file at 1732486614421Flushing 1588230740/rep_barrier: creating writer at 1732486614828 (+407 ms)Flushing 1588230740/rep_barrier: appending metadata at 1732486614843 (+15 ms)Flushing 1588230740/rep_barrier: closing flushed file at 1732486614843Flushing 1588230740/table: creating writer at 1732486615250 (+407 ms)Flushing 1588230740/table: appending metadata at 1732486615264 (+14 ms)Flushing 1588230740/table: closing flushed file at 1732486615264Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@61c28cef: reopening flushed file at 1732486615671 (+407 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@93540c7: reopening flushed file at 1732486615674 (+3 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@11a5c517: reopening flushed file at 1732486615677 (+3 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@56cc4f39: reopening flushed file at 1732486615679 (+2 ms)Finished flush of dataSize ~19.91 KB/20384, heapSize ~35.14 KB/35984, currentSize=0 B/0 for 1588230740 in 1706ms, sequenceid=90, compaction requested=false at 1732486615682 (+3 ms)Writing region close event to WAL at 1732486615684 (+2 ms)Running coprocessor post-close hooks at 1732486615686 (+2 ms)Closed at 1732486615687 (+1 ms) 2024-11-24T22:16:55,687 DEBUG [RS_CLOSE_META-regionserver/bd6592c37bb2:0-0 {event_type=M_RS_CLOSE_META}] handler.CloseRegionHandler(117): Closed hbase:meta,,1.1588230740 2024-11-24T22:16:55,778 INFO [RS:0;bd6592c37bb2:40087 {}] regionserver.HRegionServer(976): stopping server bd6592c37bb2,40087,1732486423713; all regions closed. 2024-11-24T22:16:55,782 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45929 is added to blk_1073741834_1010 (size=25344) 2024-11-24T22:16:55,784 DEBUG [RS:0;bd6592c37bb2:40087 {}] wal.AbstractFSWAL(1256): Moved 1 WAL file(s) to /user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/oldWALs 2024-11-24T22:16:55,785 INFO [RS:0;bd6592c37bb2:40087 {}] wal.AbstractFSWAL(1259): Closed WAL: AsyncFSWAL bd6592c37bb2%2C40087%2C1732486423713.meta:.meta(num 1732486426018) 2024-11-24T22:16:55,786 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45929 is added to blk_1073741833_1009 (size=22380688) 2024-11-24T22:16:55,788 DEBUG [RS:0;bd6592c37bb2:40087 {}] wal.AbstractFSWAL(1256): Moved 1 WAL file(s) to /user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/oldWALs 2024-11-24T22:16:55,788 INFO [RS:0;bd6592c37bb2:40087 {}] wal.AbstractFSWAL(1259): Closed WAL: AsyncFSWAL bd6592c37bb2%2C40087%2C1732486423713:(num 1732486425471) 2024-11-24T22:16:55,788 DEBUG [RS:0;bd6592c37bb2:40087 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-24T22:16:55,788 INFO [RS:0;bd6592c37bb2:40087 {}] regionserver.LeaseManager(133): Closed leases 2024-11-24T22:16:55,788 INFO [RS:0;bd6592c37bb2:40087 {}] hbase.HBaseServerBase(438): Shutdown chores and chore service 2024-11-24T22:16:55,789 INFO [RS:0;bd6592c37bb2:40087 {}] hbase.ChoreService(370): Chore service for: regionserver/bd6592c37bb2:0 had [ScheduledChore name=CompactionThroughputTuner, period=60000, unit=MILLISECONDS, ScheduledChore name=ReplicationSinkStatistics, period=300000, unit=MILLISECONDS, ScheduledChore name=ReplicationSourceStatistics, period=300000, unit=MILLISECONDS] on shutdown 2024-11-24T22:16:55,789 INFO [RS:0;bd6592c37bb2:40087 {}] hbase.HBaseServerBase(448): Shutdown executor service 2024-11-24T22:16:55,789 INFO [regionserver/bd6592c37bb2:0.logRoller {}] wal.AbstractWALRoller(249): LogRoller exiting. 2024-11-24T22:16:55,789 INFO [RS:0;bd6592c37bb2:40087 {}] ipc.NettyRpcServer(345): Stopping server on /172.17.0.2:40087 2024-11-24T22:16:55,794 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:40087-0x1014c0ac8d60001, quorum=127.0.0.1:59994, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/rs/bd6592c37bb2,40087,1732486423713 2024-11-24T22:16:55,794 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:37701-0x1014c0ac8d60000, quorum=127.0.0.1:59994, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/rs 2024-11-24T22:16:55,794 INFO [RS:0;bd6592c37bb2:40087 {}] hbase.HBaseServerBase(479): Close zookeeper 2024-11-24T22:16:55,796 INFO [RegionServerTracker-0 {}] master.RegionServerTracker(171): RegionServer ephemeral node deleted, processing expiration [bd6592c37bb2,40087,1732486423713] 2024-11-24T22:16:55,802 DEBUG [RegionServerTracker-0 {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/draining/bd6592c37bb2,40087,1732486423713 already deleted, retry=false 2024-11-24T22:16:55,802 INFO [RegionServerTracker-0 {}] master.ServerManager(688): Cluster shutdown set; bd6592c37bb2,40087,1732486423713 expired; onlineServers=0 2024-11-24T22:16:55,802 INFO [RegionServerTracker-0 {}] master.HMaster(3321): ***** STOPPING master 'bd6592c37bb2,37701,1732486422992' ***** 2024-11-24T22:16:55,802 INFO [RegionServerTracker-0 {}] master.HMaster(3323): STOPPED: Cluster shutdown set; onlineServer=0 2024-11-24T22:16:55,803 INFO [M:0;bd6592c37bb2:37701 {}] hbase.HBaseServerBase(455): Close async cluster connection 2024-11-24T22:16:55,803 INFO [M:0;bd6592c37bb2:37701 {}] hbase.HBaseServerBase(438): Shutdown chores and chore service 2024-11-24T22:16:55,803 DEBUG [M:0;bd6592c37bb2:37701 {}] cleaner.LogCleaner(198): Cancelling LogCleaner 2024-11-24T22:16:55,803 DEBUG [M:0;bd6592c37bb2:37701 {}] cleaner.HFileCleaner(335): Stopping file delete threads 2024-11-24T22:16:55,803 WARN [OldWALsCleaner-0 {}] cleaner.LogCleaner(186): Interrupted while cleaning old WALs, will try to clean it next round. Exiting. 2024-11-24T22:16:55,803 DEBUG [master/bd6592c37bb2:0:becomeActiveMaster-HFileCleaner.large.0-1732486425138 {}] cleaner.HFileCleaner(306): Exit Thread[master/bd6592c37bb2:0:becomeActiveMaster-HFileCleaner.large.0-1732486425138,5,FailOnTimeoutGroup] 2024-11-24T22:16:55,803 DEBUG [master/bd6592c37bb2:0:becomeActiveMaster-HFileCleaner.small.0-1732486425140 {}] cleaner.HFileCleaner(306): Exit Thread[master/bd6592c37bb2:0:becomeActiveMaster-HFileCleaner.small.0-1732486425140,5,FailOnTimeoutGroup] 2024-11-24T22:16:55,803 INFO [M:0;bd6592c37bb2:37701 {}] hbase.ChoreService(370): Chore service for: master/bd6592c37bb2:0 had [ScheduledChore name=FlushedSequenceIdFlusher, period=10800000, unit=MILLISECONDS] on shutdown 2024-11-24T22:16:55,803 INFO [M:0;bd6592c37bb2:37701 {}] hbase.HBaseServerBase(448): Shutdown executor service 2024-11-24T22:16:55,803 DEBUG [M:0;bd6592c37bb2:37701 {}] master.HMaster(1795): Stopping service threads 2024-11-24T22:16:55,803 INFO [M:0;bd6592c37bb2:37701 {}] procedure2.RemoteProcedureDispatcher(119): Stopping procedure remote dispatcher 2024-11-24T22:16:55,804 INFO [M:0;bd6592c37bb2:37701 {}] procedure2.ProcedureExecutor(723): Stopping 2024-11-24T22:16:55,804 ERROR [M:0;bd6592c37bb2:37701 {}] procedure2.ProcedureExecutor(763): There are still active thread in group java.lang.ThreadGroup[name=PEWorkerGroup,maxpri=10], see STDOUT java.lang.ThreadGroup[name=PEWorkerGroup,maxpri=10] Thread[IPC Client (1102635514) connection to localhost/127.0.0.1:43173 from jenkins,5,PEWorkerGroup] Thread[IPC Parameter Sending Thread for localhost/127.0.0.1:43173,5,PEWorkerGroup] Thread[HFileArchiver-6,5,PEWorkerGroup] 2024-11-24T22:16:55,804 INFO [M:0;bd6592c37bb2:37701 {}] region.RegionProcedureStore(113): Stopping the Region Procedure Store, isAbort=false 2024-11-24T22:16:55,804 DEBUG [normalizer-worker-0 {}] normalizer.RegionNormalizerWorker(193): interrupt detected. terminating. 2024-11-24T22:16:55,805 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:37701-0x1014c0ac8d60000, quorum=127.0.0.1:59994, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/master 2024-11-24T22:16:55,805 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:37701-0x1014c0ac8d60000, quorum=127.0.0.1:59994, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-24T22:16:55,805 DEBUG [M:0;bd6592c37bb2:37701 {}] zookeeper.ZKUtil(347): master:37701-0x1014c0ac8d60000, quorum=127.0.0.1:59994, baseZNode=/hbase Unable to get data of znode /hbase/master because node does not exist (not an error) 2024-11-24T22:16:55,805 WARN [M:0;bd6592c37bb2:37701 {}] master.ActiveMasterManager(344): Failed get of master address: java.io.IOException: Can't get master address from ZooKeeper; znode data == null 2024-11-24T22:16:55,806 INFO [M:0;bd6592c37bb2:37701 {}] master.ServerManager(1139): Writing .lastflushedseqids file at: hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/.lastflushedseqids 2024-11-24T22:16:55,815 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45929 is added to blk_1073742498_1674 (size=72) 2024-11-24T22:16:55,896 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:40087-0x1014c0ac8d60001, quorum=127.0.0.1:59994, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-11-24T22:16:55,896 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:40087-0x1014c0ac8d60001, quorum=127.0.0.1:59994, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-11-24T22:16:55,897 INFO [RS:0;bd6592c37bb2:40087 {}] hbase.HBaseServerBase(486): Close table descriptors 2024-11-24T22:16:55,897 INFO [RS:0;bd6592c37bb2:40087 {}] regionserver.HRegionServer(1031): Exiting; stopping=bd6592c37bb2,40087,1732486423713; zookeeper connection closed. 2024-11-24T22:16:55,897 INFO [Shutdown of org.apache.hadoop.hbase.fs.HFileSystem@be496f2 {}] hbase.SingleProcessHBaseCluster$SingleFileSystemShutdownThread(211): Hook closing fs=org.apache.hadoop.hbase.fs.HFileSystem@be496f2 2024-11-24T22:16:55,898 INFO [Time-limited test {}] util.JVMClusterUtil(335): Shutdown of 1 master(s) and 1 regionserver(s) complete 2024-11-24T22:16:56,216 INFO [M:0;bd6592c37bb2:37701 {}] assignment.AssignmentManager(395): Stopping assignment manager 2024-11-24T22:16:56,216 INFO [M:0;bd6592c37bb2:37701 {}] region.MasterRegion(195): Closing local region {ENCODED => 1595e783b53d99cd5eef43b6debb2682, NAME => 'master:store,,1.1595e783b53d99cd5eef43b6debb2682.', STARTKEY => '', ENDKEY => ''}, isAbort=false 2024-11-24T22:16:56,216 DEBUG [M:0;bd6592c37bb2:37701 {}] regionserver.HRegion(1722): Closing 1595e783b53d99cd5eef43b6debb2682, disabling compactions & flushes 2024-11-24T22:16:56,216 INFO [M:0;bd6592c37bb2:37701 {}] regionserver.HRegion(1755): Closing region master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-24T22:16:56,216 DEBUG [M:0;bd6592c37bb2:37701 {}] regionserver.HRegion(1776): Time limited wait for close lock on master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-24T22:16:56,216 DEBUG [M:0;bd6592c37bb2:37701 {}] regionserver.HRegion(1843): Acquired close lock on master:store,,1.1595e783b53d99cd5eef43b6debb2682. after waiting 0 ms 2024-11-24T22:16:56,216 DEBUG [M:0;bd6592c37bb2:37701 {}] regionserver.HRegion(1853): Updates disabled for region master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-24T22:16:56,216 INFO [M:0;bd6592c37bb2:37701 {}] regionserver.HRegion(2902): Flushing 1595e783b53d99cd5eef43b6debb2682 4/4 column families, dataSize=815.43 KB heapSize=1002.36 KB 2024-11-24T22:16:56,231 DEBUG [M:0;bd6592c37bb2:37701 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/info/e82d479a8ac14e7c8a1ffd664079d936 is 82, key is hbase:meta,,1/info:regioninfo/1732486426124/Put/seqid=0 2024-11-24T22:16:56,234 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45929 is added to blk_1073742499_1675 (size=5672) 2024-11-24T22:16:56,635 INFO [M:0;bd6592c37bb2:37701 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=504 B at sequenceid=2321 (bloomFilter=true), to=hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/info/e82d479a8ac14e7c8a1ffd664079d936 2024-11-24T22:16:56,658 DEBUG [M:0;bd6592c37bb2:37701 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/proc/826a6de555d442ba83821710dcbddfc4 is 2479, key is \x00\x00\x00\x00\x00\x00\x00\x5C/proc:d/1732486524676/Put/seqid=0 2024-11-24T22:16:56,664 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45929 is added to blk_1073742500_1676 (size=45699) 2024-11-24T22:16:56,664 INFO [M:0;bd6592c37bb2:37701 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=814.88 KB at sequenceid=2321 (bloomFilter=true), to=hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/proc/826a6de555d442ba83821710dcbddfc4 2024-11-24T22:16:56,667 INFO [M:0;bd6592c37bb2:37701 {}] regionserver.StoreFileReader(518): Loaded Delete Family Bloom (CompoundBloomFilter) metadata for 826a6de555d442ba83821710dcbddfc4 2024-11-24T22:16:56,686 DEBUG [M:0;bd6592c37bb2:37701 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/rs/484ba1601cb442e08884dd7b78276a8d is 69, key is bd6592c37bb2,40087,1732486423713/rs:state/1732486425242/Put/seqid=0 2024-11-24T22:16:56,702 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45929 is added to blk_1073742501_1677 (size=5156) 2024-11-24T22:16:56,703 INFO [M:0;bd6592c37bb2:37701 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=65 B at sequenceid=2321 (bloomFilter=true), to=hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/rs/484ba1601cb442e08884dd7b78276a8d 2024-11-24T22:16:56,709 DEBUG [M:0;bd6592c37bb2:37701 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/info/e82d479a8ac14e7c8a1ffd664079d936 as hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/info/e82d479a8ac14e7c8a1ffd664079d936 2024-11-24T22:16:56,714 INFO [M:0;bd6592c37bb2:37701 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/info/e82d479a8ac14e7c8a1ffd664079d936, entries=8, sequenceid=2321, filesize=5.5 K 2024-11-24T22:16:56,714 DEBUG [M:0;bd6592c37bb2:37701 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/proc/826a6de555d442ba83821710dcbddfc4 as hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/proc/826a6de555d442ba83821710dcbddfc4 2024-11-24T22:16:56,719 INFO [M:0;bd6592c37bb2:37701 {}] regionserver.StoreFileReader(518): Loaded Delete Family Bloom (CompoundBloomFilter) metadata for 826a6de555d442ba83821710dcbddfc4 2024-11-24T22:16:56,719 INFO [M:0;bd6592c37bb2:37701 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/proc/826a6de555d442ba83821710dcbddfc4, entries=172, sequenceid=2321, filesize=44.6 K 2024-11-24T22:16:56,720 DEBUG [M:0;bd6592c37bb2:37701 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/rs/484ba1601cb442e08884dd7b78276a8d as hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/rs/484ba1601cb442e08884dd7b78276a8d 2024-11-24T22:16:56,727 INFO [M:0;bd6592c37bb2:37701 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43173/user/jenkins/test-data/31fbeeb8-eb97-c32d-3fc0-a8b02ba921cc/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/rs/484ba1601cb442e08884dd7b78276a8d, entries=1, sequenceid=2321, filesize=5.0 K 2024-11-24T22:16:56,728 INFO [M:0;bd6592c37bb2:37701 {}] regionserver.HRegion(3140): Finished flush of dataSize ~815.43 KB/835001, heapSize ~1002.06 KB/1026112, currentSize=0 B/0 for 1595e783b53d99cd5eef43b6debb2682 in 512ms, sequenceid=2321, compaction requested=false 2024-11-24T22:16:56,757 INFO [M:0;bd6592c37bb2:37701 {}] regionserver.HRegion(1973): Closed master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-24T22:16:56,757 DEBUG [M:0;bd6592c37bb2:37701 {}] regionserver.HRegion(1676): Region close journal for 1595e783b53d99cd5eef43b6debb2682: Waiting for close lock at 1732486616216Disabling compacts and flushes for region at 1732486616216Disabling writes for close at 1732486616216Obtaining lock to block concurrent updates at 1732486616216Preparing flush snapshotting stores in 1595e783b53d99cd5eef43b6debb2682 at 1732486616216Finished memstore snapshotting master:store,,1.1595e783b53d99cd5eef43b6debb2682., syncing WAL and waiting on mvcc, flushsize=dataSize=835001, getHeapSize=1026352, getOffHeapSize=0, getCellsCount=2518 at 1732486616216Flushing stores of master:store,,1.1595e783b53d99cd5eef43b6debb2682. at 1732486616217 (+1 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/info: creating writer at 1732486616217Flushing 1595e783b53d99cd5eef43b6debb2682/info: appending metadata at 1732486616231 (+14 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/info: closing flushed file at 1732486616231Flushing 1595e783b53d99cd5eef43b6debb2682/proc: creating writer at 1732486616638 (+407 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/proc: appending metadata at 1732486616657 (+19 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/proc: closing flushed file at 1732486616657Flushing 1595e783b53d99cd5eef43b6debb2682/rs: creating writer at 1732486616667 (+10 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/rs: appending metadata at 1732486616685 (+18 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/rs: closing flushed file at 1732486616685Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@3cb0cf58: reopening flushed file at 1732486616707 (+22 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@207a3284: reopening flushed file at 1732486616714 (+7 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@5f53bd0b: reopening flushed file at 1732486616719 (+5 ms)Finished flush of dataSize ~815.43 KB/835001, heapSize ~1002.06 KB/1026112, currentSize=0 B/0 for 1595e783b53d99cd5eef43b6debb2682 in 512ms, sequenceid=2321, compaction requested=false at 1732486616728 (+9 ms)Writing region close event to WAL at 1732486616756 (+28 ms)Closed at 1732486616757 (+1 ms) 2024-11-24T22:16:56,761 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45929 is added to blk_1073741830_1006 (size=987073) 2024-11-24T22:16:56,762 INFO [M:0;bd6592c37bb2:37701 {}] flush.MasterFlushTableProcedureManager(90): stop: server shutting down. 2024-11-24T22:16:56,762 INFO [M:0;bd6592c37bb2:37701 {}] ipc.NettyRpcServer(345): Stopping server on /172.17.0.2:37701 2024-11-24T22:16:56,763 INFO [M:0;bd6592c37bb2:37701 {}] hbase.HBaseServerBase(479): Close zookeeper 2024-11-24T22:16:56,763 INFO [master:store-WAL-Roller {}] wal.AbstractWALRoller(249): LogRoller exiting. 2024-11-24T22:16:56,865 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:37701-0x1014c0ac8d60000, quorum=127.0.0.1:59994, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-11-24T22:16:56,865 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:37701-0x1014c0ac8d60000, quorum=127.0.0.1:59994, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-11-24T22:16:56,868 INFO [M:0;bd6592c37bb2:37701 {}] hbase.HBaseServerBase(486): Close table descriptors 2024-11-24T22:16:56,878 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.w.WebAppContext@4269bd31{datanode,/,null,STOPPED}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/datanode} 2024-11-24T22:16:56,882 INFO [Time-limited test {}] server.AbstractConnector(383): Stopped ServerConnector@353d64f2{HTTP/1.1, (http/1.1)}{localhost:0} 2024-11-24T22:16:56,882 INFO [Time-limited test {}] session.HouseKeeper(149): node0 Stopped scavenging 2024-11-24T22:16:56,882 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@435f9f96{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,STOPPED} 2024-11-24T22:16:56,882 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@4fd7fa7c{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/c25542eb-68a7-66d5-8606-05e68b15f363/hadoop.log.dir/,STOPPED} 2024-11-24T22:16:56,886 ERROR [Command processor {}] datanode.BPServiceActor$CommandProcessingThread(1416): Command processor encountered interrupt and exit. 2024-11-24T22:16:56,886 WARN [BP-2007615641-172.17.0.2-1732486419886 heartbeating to localhost/127.0.0.1:43173 {}] datanode.IncrementalBlockReportManager(160): IncrementalBlockReportManager interrupted 2024-11-24T22:16:56,886 WARN [Command processor {}] datanode.BPServiceActor$CommandProcessingThread(1400): Ending command processor service for: Thread[Command processor,5,FailOnTimeoutGroup] 2024-11-24T22:16:56,886 WARN [BP-2007615641-172.17.0.2-1732486419886 heartbeating to localhost/127.0.0.1:43173 {}] datanode.BPServiceActor(925): Ending block pool service for: Block pool BP-2007615641-172.17.0.2-1732486419886 (Datanode Uuid 1c5ad8e8-4392-4afa-9943-4f874d844787) service to localhost/127.0.0.1:43173 2024-11-24T22:16:56,888 WARN [refreshUsed-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/c25542eb-68a7-66d5-8606-05e68b15f363/cluster_c8a06051-0388-dc48-080e-6657ef7cdb74/data/data1/current/BP-2007615641-172.17.0.2-1732486419886 {}] fs.CachingGetSpaceUsed$RefreshThread(231): Thread Interrupted waiting to refresh disk information: sleep interrupted 2024-11-24T22:16:56,889 WARN [refreshUsed-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/c25542eb-68a7-66d5-8606-05e68b15f363/cluster_c8a06051-0388-dc48-080e-6657ef7cdb74/data/data2/current/BP-2007615641-172.17.0.2-1732486419886 {}] fs.CachingGetSpaceUsed$RefreshThread(231): Thread Interrupted waiting to refresh disk information: sleep interrupted 2024-11-24T22:16:56,889 WARN [Time-limited test {}] datanode.DataSetLockManager(261): not open lock leak check func 2024-11-24T22:16:56,897 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.w.WebAppContext@4ded99d3{hdfs,/,null,STOPPED}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/hdfs} 2024-11-24T22:16:56,898 INFO [Time-limited test {}] server.AbstractConnector(383): Stopped ServerConnector@557285fa{HTTP/1.1, (http/1.1)}{localhost:0} 2024-11-24T22:16:56,898 INFO [Time-limited test {}] session.HouseKeeper(149): node0 Stopped scavenging 2024-11-24T22:16:56,898 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@93c62df{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,STOPPED} 2024-11-24T22:16:56,898 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@6b23711f{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/c25542eb-68a7-66d5-8606-05e68b15f363/hadoop.log.dir/,STOPPED} 2024-11-24T22:16:56,916 INFO [Time-limited test {}] zookeeper.MiniZooKeeperCluster(347): Shutdown MiniZK cluster with all ZK servers 2024-11-24T22:16:57,049 INFO [Time-limited test {}] hbase.HBaseTestingUtil(1026): Minicluster is down